{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tqdm\n",
    "import string\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import *\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import recall_score, precision_score, roc_auc_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1344, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ride_review</th>\n",
       "      <th>ride_rating</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I completed running New York Marathon requeste...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>My appointment time auto repairs required earl...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Whether I using Uber ride service Uber Eats or...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Why hard understand I trying retrieve Uber cab...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I South Beach FL I staying major hotel ordered...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         ride_review  ride_rating  sentiment\n",
       "0  I completed running New York Marathon requeste...          1.0          0\n",
       "1  My appointment time auto repairs required earl...          1.0          0\n",
       "2  Whether I using Uber ride service Uber Eats or...          1.0          0\n",
       "3  Why hard understand I trying retrieve Uber cab...          1.0          0\n",
       "4  I South Beach FL I staying major hotel ordered...          1.0          0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### READ DATA ###\n",
    "\n",
    "df = pd.read_csv('Uber_Ride_Reviews.csv')\n",
    "\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2254eadb988>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4MAAAF7CAYAAABoyV6nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeZwkdX3/8deHXZZDbliQuxEQRaNIEBbijUSwRTQRzygqiggqoFFbjWGMUVtNPH8qohBA8T4i0ngQ8UwEOTy4VBAbWM7lGkCBvT6/P6oWhmV3555v9/Tr+Xj0Y6ara6reU7BT/e6q+lZkJpIkSZKkwbJW6QCSJEmSpJlnGZQkSZKkAWQZlCRJkqQBZBmUJEmSpAFkGZQkSZKkAWQZlCRJkqQBZBnUrBERJ0TEu9fwekbELjOZaXUi4tKIeFqhde8QEXdHxJwpWt792z0inhYRC6diufXynhwRf5iq5UmSZo/R9vu9birzr7xvj4ifRMRrpmLZ9fK+FxGHTdXy1Dssg+obEdGNiHvqP3Y3RsQpEbHBitcz88jMfO80rPeVEbGsXu+dEfHbiHjOOH7+lIj495HTMvMxmfmTac56d0T8OSL+KyIeOWLd12TmBpm5bAzL+sVo65zK7b5yYc/Mn2fmblOxbEnS9IuIJ0XE/0XEcETcFhH/GxFPnILlPmSfNF37/TFkGYqIL44yz4r3LHdFxB31NjkyIu5/7z3W/PWynrmmeca6bx+LVf1+mXlQZp462WWr91gG1W8OzswNgD2AJwDvmKH1/rJe7ybAp4GvRMQmM7Tu8VqRdWPgmcA9wIUR8dipXtFUHV2UJPW/iNgIOBP4JLAZsC3wHuC+krkKOjgzNwR2BNrA24GTpnolETF3qpepwWEZVF/KzBuBH1CVQuChR+Ai4q0RcUNEXB8Rrx758xGxTkT8R0RcExE31adqrDeG9S4HvgA8DNh1xPK+Xh+tHI6In0XEY+rpRwAvA95WH6n7bj39/k/56k/gvhYRp9WfIF4aEXuNWPaeEfHr+rWvR8RXVz7SuJqsyzLzT5l5FPBTYKheXqM+Aje3fv7KiLiqXv6fI+JlEfFo4ARg3zr3HSO28Wci4qyI+Avw9FUd+YyId0bELfXv+bIR0x902srIT3oj4mf15N/W63xRrHTaaUQ8ul7GHfV2eu6I106JiE9FRKf+Xc6LiJ1H206SpCnzSIDM/HK9D7onM3+Ymb9bMUNEvDoiLo+I2yPiBxGx44jXsj56dkX9+qeisqZ90r/X3z8tIhZGxNsi4uZ6//+8iHh2RPyxPkr5zhHrWisiWhHxp4i4td4Pb1a/tmI/eVj9PuGWiHhX/dqBwDuBF9VZfjvaRsnM4cw8A3gRcNiKD2dXyr9FRJxZ799ui4if1xm/AOwAfLde39tG5Ds8Iq4Bzll5317bOSJ+Vb83+c6I3+8hl3SseF+yut9v5P67zvUvEXF1va1Pi4iNR9t26k2WQfWliNgOOAi4cjWvHwj8M3AAVWlb+fSKD1LttPYAdqH69PJfx7DeOcCrgCXA1SNe+l69ni2Bi4DTATLzxPr7D9Wnbxy8mkU/F/gK1ZHHM4D/V69vHvBt4BSqT1m/DDx/tJyr8C3gyav4fR4GfAI4qP70cj/gN5l5OXAk9VHGzBx5FPSlwPuADYFVnUb6cGALqm16GHBiRIx6qmdmPqX+9vH1Or+6Uta1ge8CP6Tazm8ETl9p2S+h+hR6U6r/N9432nolSVPmj8CyiDg1Ig6KiE1HvhgRz6MqGv8AzAd+TrVfG+k5wBOBxwMvBJ41yj5ppIcD6/LAPv1zwD8Bf0u1D/zXiHhEPe+bgOcBTwW2AW4HPrXS8p4E7AbsX//sozPz+8D7ga/WWR4/tk0DmfkrYCGr2B8Db6lfmw9sRbWdMjNfDlxDfWZUZn5oxM88FXg08KzVrPIVwKvr328p1f5+tIxj+f1eWT+eDjwC2ID6fcsID9l2o61bZVgG1W/+OyLuAq4FbgaOX818LwT+KzMvycy/UB8VA4iIAF4LHJeZt2XmXVR/+F68hvUuqD+JvBf4D+CfMvPmFS9m5smZeVdm3lev6/ErPiUbo19k5ln1uf5foNoJAiwA5gKfyMwlmfkt4FfjWO4K11OVyVVZDjw2ItbLzBsy89JRlvWdzPzfzFyemfeuZp53Z+Z9mflToEP132OyFlDtcNqZuTgzz6E6HeklI+b5Vmb+KjOXUpXwPVaxHEnSNMjMO6lKQFIVsUURcUZEbFXP8jrgA5l5ef13+v3AHiOPDlL9jb8jM68Bfsz4/o4vAd6XmUuoPmDdAvh4vX++FLgUeNyILO/KzIUj9t0vWOnI2nvqo5u/BX7LA/vmyVjd/ngJsDWwY72//3lm5ijLGsrMv2TmPat5/Qsj3ge9G3hhTM3lHS8DPpKZV2Xm3VSX7Lx4BradpoFlUP3mefURrKcBj6L6Q78q21AVxhVGHsWbD6xPdR3dHXXJ+349fXXOrT+J3JTqyN39n+pFxJyIaNenmtwJdOuXVpdtVW4c8f1fgXXrP6rbANettEO4lvHbFrht5Yn1DuJFVJ+43lCfYvmoUZY12vpvr5e7wtVUv8dkbQNcW5+qO3LZ2454vvJ23ABJ0oypi94rM3M74LFUf7s/Vr+8I/DxEfve24Bg6v6O3zpiAJUVBemmEa/fM2J5OwLfHpHlcmAZ1VG5qciyOqvcHwMfpjqj5YdRXbrRGsOyRtsfr/w+aG3G995kdbbhwe+rrqb64Hq6t52mgWVQfak+4nQK1VG6VbkB2H7E8x1GfH8L1Q7hMZm5Sf3YuB50ZbT13g0cBbw8Ip5QT34pcAjVqagbA416eqz4sVF/odW7Adi2Ppq5wvarm3kNnk91Os5DZOYPMvMAqk8kf0/1aS6sPvdov8+m9emnK+xA9UkowF+oivgKDx9lWSNdD2wfI0Ziq5d93TiWIUmaIZn5e6p99YoBzK4FXjdi37tJZq6Xmf83lsVNcbxrqS6RGJll3cwcyz5lQlmiGlV1W1ZxiUV99PItmfkI4GDgzRGx/yjrGy3Hyu+DllC9B3rQvrg+WjjyA/HRlns9VZkeueylPLh4q09YBtXPPgYcEBGrOoXka8ArI2L3iFifEaeT1keWPgd8NCK2BIiIbSNidefcP0hm3gp8ngeuMdyQaqS0W6n+uL5/pR+5ieqc+on4JdUnlW+IiLkRcQiw91h+sD5iuVNEfJLqSOp7VjHPVhHx3Lq83QfcXa9vRe7t6usWx+s9ETEvIp5Mdf3H1+vpvwH+ISLWj+oWEoev9HNr2lbnUe3A3hYRa0d1n8aDqU4FkiQVFhGPioi31Nf1ExHbU53Kf249ywnAO+KBQdY2johDx7j4yeyTVuUE4H0rTlGNiPn1PnasWRorfTi5WhGxUVS3pPoK8MXMvHgV8zwnInapP/y9k2pfPHJ/PJH3Ef804n3QvwHfqI+c/pHqDKRmfT3+vwDrjOP3+zJwXP0eYwMeuMZw6QQyqjDLoPpWZi4CTqM6D37l175HVRbPoTrt4pyVZnl7Pf3c+tTO/6G60HmsPgY8OyIeV2e4muoI1WU8sNNb4SRg9/pUlP8exzrIzMVUF9ofDtxBdSH8max5mO59I+Juqp3JT4CNgCeuaudD9TfgLVSf8t1GdTH6UfVr51BdX3FjRNwyjtg3Ul2Ifz3VdXtH1p8OA3wUWEy1ozm1fn2kIeDUels96DrDels8l2rgoFuobvHxihHLliSVdRewD3BeVCNOnwtcQrWfITO/TTWA21fqfe8lVH/Tx2Ki+6TV+TjVZR8/rMciOLfOPhYrPuC8NSIuWsN83x0xzsG7gI9QDUK3KrtSvRe5m+qD4E/nA/cj/gDwL/W+8Z/HmBGqMQhOodovr0s1aA6ZOUy1r/881XuXv1ANXjPW3+/ketk/A/5MNZ7CG8eRSz0kRr82VVIviYjzgBMy879KZ5EkSVL/8sig1OMi4qkR8fD6NNHDqEZC+37pXJIkSepvc0efRVJhu1FdA7kB8CfgBZl5Q9lIkiRJ6neeJipJkiRJA8jTRCVJkiRpAFkGJUmSJGkA9fQ1g1tssUU2Go3SMSRJs8SFF154S2bOH31OSZJmv54ug41GgwsuuKB0DEnSLBERV5fOIElSr/A0UUmSJEkaQJZBSZIkSRpAlkFJkiRJGkCWQUmSJEkaQJZBSZIkSRpAlkFJkiRJGkCWQUmSJEkaQJZBSZIkSRpAlkFJkiRJGkCWQUmSJEkaQJZBSZIkSRpAlkFJkiRJGkCWQUmSJEkaQHNLB5hpjVandIQ16rabpSNIkiRJGgAeGZQkSZKkAWQZlCRJkqQBZBmUJEmSpAFkGZQkSZKkAWQZlCRJkqQBZBmUJEmSpAFkGZQkSZKkAWQZlCRJkqQBNGoZjIiTI+LmiLhkxLTNIuLsiLii/rppPT0i4hMRcWVE/C4i9hzxM4fV818REYdNz68jSZIkSRqLsRwZPAU4cKVpLeBHmbkr8KP6OcBBwK714wjgM1CVR+B4YB9gb+D4FQVSkiRJkjTzRi2Dmfkz4LaVJh8CnFp/fyrwvBHTT8vKucAmEbE18Czg7My8LTNvB87moQVTkiRJkjRDJnrN4FaZeQNA/XXLevq2wLUj5ltYT1vddEmSJElSAVM9gEysYlquYfpDFxBxRERcEBEXLFq0aErDSZIkSZIqEy2DN9Wnf1J/vbmevhDYfsR82wHXr2H6Q2TmiZm5V2buNX/+/AnGkyRJkiStyUTL4BnAihFBDwO+M2L6K+pRRRcAw/VppD8A/j4iNq0Hjvn7epokSZIkqYC5o80QEV8GngZsERELqUYFbQNfi4jDgWuAQ+vZzwKeDVwJ/BV4FUBm3hYR7wXOr+f7t8xceVAaSZIkSdIMGbUMZuZLVvPS/quYN4GjV7Ock4GTx5VOkiRJKqzR6qwNPBzYDNh0xGNj4GHA+iMe6wLLgPvqx70rfR35/V3AdVQDLd7UbTdXOaaGNF1GLYOSJEnSbNdodTYDdgYeUT9Gfr8dMGeaIyxptDorxtq4tv66cMTzy7rt5t3TnEEDxjIoSZKkgdFoddYB9gQWAHsDuwE7AZuUzAWsDexYP1YlG63OFcCvgYtWfO22m7fOUD7NQpZBSZIkzVqNVucRVMVvxePxwLyioSYmgEfWjxetmNhoda6lKocrCuJ53Xbz5lUuQVqJZVCSJEmzRqPV2RN4FrAfsA8w2+9Vtn39OKR+no1W5xLgf+rHzzy9VKtjGZQkSVLfarQ684BnAM8FnsOD7209iAL4m/pxHNW1iL8EOkCn225eWjKceotlUJIkSX2l0epsDjSpCuDfAxuWTdTT1gaeUj8+2Gh1ulTF8DvAOd12c1nBbCrMMihJkqSe12h1tgVeTHU65H5M/+ies1WD6lZwRwPXN1qd04FTPWI4mCyDkiRJ6kn1yJ+HAK8GDgDWKpto1tkGeCvw1karcxFwGvClbru5qGwszRTLoCRJknpKo9X5G+AI4KVUN3rX9Nuzfny40ep8HzgV+G633VxcNpamk2VQkiRJxdUDwbwAeD3wpMJxBtnawMH147ZGq/MF4KPddvPqsrE0HSyDkiRJKqbR6mwFHAMcDmxZOI4ebDOq/zZHN1qdrwIf6rabvyucSVPIMihJkqQZ12h1tgbeBrwOWK9wHK3ZXOBlwMvqU0g/2G03f1I2kqaCZVCSJEkzph4VtAW8Bli3cByN34HAgY1W51fAB4H/7rabywtn0gRZBiVJkjTtGq3ODsA7gFcB6xSOo8nbG/gm8MdGq/NhqttTLCmcSePk8LySJEmaNo1Wp9FodU4ErgSOxCI42zwS+BxwSaPVObh0GI2PRwYlSZI05RqtzqbAe6muCfQ95+z3SOCMRqvzI+C4brt5celAGp3/MCVJkjRlGq1OUN0k/gPA/MJxNPP2B37daHVOAt7dbTdvLh1Iq+dpopIkSZoSjVZnT+D/gM9jERxkc4AjgCsarc7bG62Opwb3KMugJEmSJqXR6mzWaHU+A5wPLCidRz1jI6ANXN5odQ4tHUYP5WmikiRJmpD6lNDDqU4J3aJwHPWunYCvNVqds4HXdtvNq0sHUsUjg5IkSRq3RquzB/BLqpEkLYIaiwOAixutzuvrDxJUmGVQkiRJY9ZodaLR6vwzcB6wT+k86jsbAp8GftRodR5ROsygswxKkiRpTBqtzjbAD4EPA/MKx1F/ezrwu0ar89rSQQaZZVCSJEmjarQ6hwC/A55ZOotmjYcBJzZanf9utDqOPluAZVCSJEmr1Wh11m+0Op8F/hvYvHQezUqHUF1L2CwdZNBYBiVJkrRKjVbnCcCFVPeMk6bTVsCZjVbnA41Wx44yQ9zQkiRJepB6kJi3AOcCjyqdRwOlRVUKNykdZBBYBiVJknS/RquzPvA14D9wkBiVcRDwq0ars3vpILOdZVCSJEkANFqdHYD/BV5QOosG3q7AufXARZomlkFJkiTRaHX+Djgf2KN0Fqm2IfDtRqsz5E3qp4dlUJIkacA1Wp2XA+cAW5bOIq0kgOOpSuGGpcPMNpZBSZKkAdZodY4HTsPrA9XbDgHOa7Q6jygdZDaxDEqSJA2gRquzdqPVOQUYKhxFGqtHAz9vtDqPLh1ktrAMSpIkDZj6dLvvAYeVziKN0zbAz+p7YGqSLIOSJEkDpC6C3wf2L51FmqAtgB83Wp39Sgfpd5ZBSZKkAdFodTYCfgD4Jlr9bmPgh41W55mlg/Qzy6AkSdIAGFEE9y2dRZoiDwPObLQ6zy0dpF9ZBiVJkma5RquzMfBDYEHpLNIUWwf4ZqPVeXHpIP3IMihJkjSLjSiC+5TOIk2TucDpjVbnNaWD9BvLoCRJ0izVaHU2Ac4G9i6dRZpmawEnNlqdl5QO0k8sg5IkSbPQiCL4xNJZpBkSwCmNVseRcsfIMihJkjTLNFqddYGzgL1KZ5Fm2Dzg241WZ4/SQfqBZVCSJGkWabQ6AZyCo4ZqcG0IfK/R6jRKB+l1lkFJkqTZ5b3Ai0qHkAp7OPCDRquzRekgvcwyKEmSNEs0Wp3DgHeVziH1iEdS3Ydw/dJBepVlUJIkaRZotDpPBU4snUPqMfsAX2u0OnNLB+lFlkFJkqQ+12h1Hgl8i2rwDEkP1gROKB2iF1kGJUmS+lij1dkc6ACblc4i9bDDG63OG0uH6DWWQUmSpD7VaHXmAd8GdimdReoD/9lodfYrHaKXWAYlSZL61yeAJ5cOIfWJtYGvN1qdrUoH6RWWQUmSpD7UaHX+EXhd6RxSn9kG+KoDylQmVQYj4riIuDQiLomIL0fEuhGxU0ScFxFXRMRXI2JePe869fMr69cbU/ELSJIk9aqIODAi/lC//2lN1XIbrc72wOemannSgHkq8G+lQ/SCCZfBiNgWeBOwV2Y+FpgDvBj4IPDRzNwVuB04vP6Rw4HbM3MX4KP1fJIkSbNSRMwBPgUcBOwOvCQidp/schutzlrAF4FNJ7ssaYC1Gq3OAaVDlDbZ00TnAutFxFxgfeAG4BnAN+rXTwWeV39/SP2c+vX9IyImuX5JkqRetTdwZWZelZmLga9QvR+arHcBT5mC5UiDLIAvDPr1gxMug5l5HfAfwDVUJXAYuBC4IzOX1rMtBLatv98WuLb+2aX1/JuvvNyIOCIiLoiICxYtWjTReJIkSaXd/96nNvJ90YTUIyEeP5llSLrfVsAX66PtA2kyp4luSvXp1k5UF2I+jOo0iJXlih9Zw2sPTMg8MTP3ysy95s+fP9F4kiRJpY3pvc9YNVqdjYEvUV2aI2lqPBM4rnSIUibTgp8J/DkzF2XmEuBbwH7AJvVpowDbAdfX3y8EtgeoX98YuG0S65ckSepl97/3qY18XzQRnwV2nFQiSavyb41WZ6fSIUqYTBm8BlgQEevX1/7tD1wG/Bh4QT3PYcB36u/PqJ9Tv35OZk740zFJkqQedz6waz3S+jyqgfbOmMiCGq3OK4EXTWE2SQ9Yn+rDloEzmWsGz6MaCOYi4OJ6WScCbwfeHBFXUl0TeFL9IycBm9fT3wxM2fDKkiRJvaYeI+ENwA+Ay4GvZeal411Oo9V5ONVI7JKmzwGNVucVpUPMtEndbDEzj+ehFzFfRTV61srz3gscOpn1SZIk9ZPMPAs4a5KL+RiwyRTEkbRmH2m0Ot/rtpsDM4rlwI6cI0mS1Osarc6BeHqoNFM2Z8COwlsGJUmSelCj1VkP+HTpHNKAeVn9IcxAsAxKkiT1pndR3cJL0sw6odHqPKx0iJlgGZQkSeoxjVZnZ+CfS+eQBtSOwHtLh5gJlkFJkqTe81FgndIhpAH2pkar85jSIaabZVCSJKmH1NcrHVw6hzTg5gDvKx1iulkGJUmSekSj1Vmb6lYSkso7pNHqLCgdYjpZBiVJknrHq4DdSoeQdL8PlA4wnSyDkiRJPaA+KvjO0jkkPcjTGq3Os0qHmC6WQUmSpN7wKqpRDCX1lvc3Wp0oHWI6WAYlSZIK86ig1NP2BA4tHWI6WAYlSZLKeyUeFZR62Xsbrc7c0iGmmmVQkiSpII8KSn3hkVSncs8qlkFJkqSyXgk0CmeQNLp/rT+8mTUsg5IkSYV4VFDqK9sBLywdYipZBiVJksp5JR4VlPrJm0oHmEqWQUmSpALqoerfXjqHpHHZu9Hq7F06xFSxDEqSJJVxALBz6RCSxm3WHB20DEqSJJXxutIBJE3IoY1W5+GlQ0wFy6AkSdIMa7Q6WwPPLZ1D0oTMA44sHWIqWAYlSZJm3quBWXcDa2mAvK7R6swrHWKyLIOSJEkzqNHqrAW8tnQOSZPycGbBbSYsg5IkSTPrWcCOpUNImrS+H0jGMihJkjSzHDhGmh2e2Gh19iwdYjIsg5IkSTOk0epsCzyndA5JU+ZlpQNMhmVQkiRp5hwOzCkdQtKUeVF9HXBf6tvgkiRJfejlpQNImlLbAk8tHWKiLIOSJEkzoNHqPAbYpXQOSVPupaUDTJRlUJIkaWY8r3QASdPiHxutTl/eN9QyKEmSNDMOKR1A0rTYFHh66RATYRmUJEmaZvUoonuVziFp2vxj6QATYRmUJEmafs8FonQISdPmkH4cVbTvAkuSJPUhrxeUZreHA/uVDjFelkFJkqRp1Gh1NqZPryeSNC4Hlw4wXpZBSZKk6XUQsHbpEJKmXd996GMZlCRJml6eIioNhj0brc6GpUOMh2VQkiRpmtQDSjyrdA5JM2IO8KTSIcbDMihJkjR9/gbYpHQISTPmqaUDjIdlUJIkafr01VECSZP2tNIBxsMyKEmSNH2eXDqApBn1t41WZ4PSIcbKMihJkjR9LIPSYJkL/F3pEGNlGZQkSZoGjVanAWxTOoekGfe00gHGyjIoSZI0PfYpHUBSEX0ziIxlUJIkaXrsXTqApCL2arQ665UOMRaWQUmSpOlhGZQG09rAo0qHGAvLoCRJ0hRrtDpzgD1L55BUzO6lA4yFZVCSJGnqPRpYv3QIScVYBiVJkiYjIh4yRPuqpvWg3UoHkFSUZVCSJGmSPjnGab1ml9IBJBXVF2VwbukAkiRJK4uIfYH9gPkR8eYRL20EzCmTalx2Lh1AUlE7N1qded12c3HpIGsyqSODEbFJRHwjIn4fEZdHxL4RsVlEnB0RV9RfN63njYj4RERcGRG/iwgvqpYkSaszD9iA6oPrDUc87gReUDDXWHlkUBpsc+iD08Une2Tw48D3M/MFETGP6kLpdwI/ysx2RLSAFvB24CBg1/qxD/AZvBmrJElahcz8KfDTiDglM68unWcCLIOSdgcuLh1iTSZcBiNiI+ApwCsBMnMxsDgiDgGeVs92KvATqjJ4CHBaZiZwbn1UcevMvGHC6SVJ0my3TkScCDQY8b4lM59RLNEoGq3OusB2pXNIKq7nrxuczJHBRwCLgP+KiMcDFwLHAFutKHiZeUNEbFnPvy1w7YifX1hPswxKkqTV+TpwAvB5YFnhLGP1CCBKh5BU3Kwug3Opbqb6xsw8LyI+TnVK6Oqs6o9iPmSmiCOAIwB22GGHScSTJEmzwNLM/EzpEOPk4DGSAHYsHWA0kxlAZiGwMDPPq59/g6oc3hQRWwPUX28eMf/2I35+O+D6lReamSdm5l6Zudf8+fMnEU+SJM0C342IoyJi63qQus0iYrPSoUbh9YKSAHq+zEy4DGbmjcC1EbFilJz9gcuAM4DD6mmHAd+pvz8DeEU9qugCYNjrBSVJ0igOA94K/B/VJSkXAhcUTTQ6y6AkgC1KBxjNZEcTfSNwej2S6FXAq6gK5tci4nDgGuDQet6zgGcDVwJ/reeVJElarczcqXSGCdimdABJPWGDRquzbrfdvLd0kNWZVBnMzN8Ae63ipf1XMW8CR09mfZIkabBExPrAm4EdMvOIiNgV2C0zzywcbU02Lh1AUs+Yz4MH0ewpk7rpvCRJ0jT7L2AxsF/9fCHw7+XijMlGpQNI6hk9fd2gZVCSJPWynTPzQ8ASgMy8h96/bYNHBiWtYBmUJEmaoMURsR717agiYmfgvrKRRmUZlLRCT5fByQ4gI0mSNJ2OB74PbB8RpwN/B7yyaKLRWQYlrdDTI4paBiVJUs/KzLMj4iJgAdXpocdk5i2FY61Wo9VZB5hXOoekntHTRwY9TVSSJPW6bYE5VCXrKRHxD4XzrIlHBSWNtHnpAGvikUFJktSzIuJk4HHApcDyenIC3yoWas0sg5JG6ukzBSyDkiSply3IzN1LhxgHy6CkkXq6b3maqCRJ6mW/jIh+KoMblg4gqafMKR1gTXq6qUqSpIF3KlUhvJHqlhIBZGY+rmys1Vo++iySBkhP962eDidJkgbeycDLgYvpj6K1uHQA9YbtYtH1G3BPr98TU9NsOXFX6QxrYhmUJEm97JrMPKN0iHGwDAqAI+ecccU/zf3RU0vnUHEbw5GlM6yW1wxKkqRe9vuI+FJEvCQi/mHFo3SoNbAMCoD3Ln35PktzrRtK51Bxy0oHWBPLoCRJ6mXrUV0r+PfAwfXjOUUTrZllUADcx7x1P7vs4CtL51BxPV0GPU1UkiT1rMx8VekM42QZ1P0+svQF+752zplXz4tlO5bOomIsg5IkSeMREW/LzA9FxCepbjL/IAda904AABxeSURBVJn5pgKxxsIyqPstY87c/1z6wuvesfaXLYODq6f/JlgGJUlSL7q8/npB0RTj19Nv/DTzTlzWXHDs3G9esV4s3rV0FhVxe+kAa2IZlCRJPSczv1t/+9fM/PrI1yLi0AKRxsoyqAdJ1lpraOkrbv/g2p8vHUVl9HQZdAAZSZLUy94xxmm9wjKoh/jqsmfsfVeud2npHCrittIB1sQjg5IkqedExEHAs4FtI+ITI17aCFhaJtWY3EuVz/dYepC3LjnivhPmfbx0DM08jwxKkiSN0/VU1wveC1w44nEG8KyCudao224msKh0DvWe7y/fZ89bc8Nfl86hGeeRQUmSpPHIzN8Cv42IL2XmktJ5xulGYOvSIdR73rTkjWufPu/9pWNoZnlkUJIkaYL2joizI+KPEXFVRPw5Iq4qHWoUN5UOoN70v8sf+9jrcvNflc6hGdXTRwYtg5IkqZedBHwEeBLwRGCv+msvswxqtY5afMwmmQ+9d6ZmLcugJEnSBA1n5vcy8+bMvHXFo3SoUdxQOoB6129zl0f+Kbf+ZekcmhH3AsOlQ6yJZVCSJPWyH0fEhyNi34jYc8WjdKhRXFs6gHrb65cct01mT4+Kq6nxZ4aGe/oosAPISJKkXrZP/XWvEdMSeEaBLGN1TekA6m1X5HaNi3Onnz8u/vzk0lk0rf5UOsBoLIOSJKlnZebTS2eYgKtLB1Dve/3iY3f5xTrH3BfBOqWzaNr0fBn0NFFJktSzImKriDgpIr5XP989Ig4vnWsUHhnUqK5j/ta/XL77uaVzaFpZBiVJkibhFOAHwDb18z8CxxZLMwbddnMYuKN0DvW+Ny15w+6Z3F06h6bNlaUDjMYyKEmSetkWmfk1YDlAZi4FlpWNNCaXlQ6g3ncLm8z/4fK/vbB0Dk0bjwxKkiRNwl8iYnOqQWOIiAX0+FDttV+XDqD+8M9LjtxjeXokeRZaDnRLhxiNZVCSJPWyNwNnADtHxP8CpwFvLBtpTH5TOoD6w108bONvLX+K/7/MPtcyNLy4dIjRWAYlSVIv2xk4CNiP6trBK+iP0dB9c68xe/eSV+29LGNR6RyaUheXDjAWlkFJktTL3p2ZdwKbAs8ETgQ+UzbSmFwC3lRcY3MP66x/6rJnXV46h6bU+aUDjIVlUJIk9bIVg8U0gRMy8zvAvIJ5xqTbbt4L/KF0DvWPDyx96YKludbC0jk0ZSyDkiRJk3RdRHwWeCFwVkSsQ/+8f/FUUY3ZEubO++TS53dL59CUsQxKkiRN0guprhU8MDPvADYD3lo20pg5oqjG5ZPLnr/vfTn3z6VzaNK6DA3fUjrEWFgGJUlSz8rMv2bmtzLzivr5DZn5w9K5xsgjgxqX5aw15wNLX3pj6RyatL44KgiWQUmSpOliGdS4nbLsWQv+muv8vnQOTYplUJIkaZB1281bqW6FIY1DxLuWvPru0ik0KZZBSZIkcXbpAOo/317+5L2Gc/3flc6hCVkGXFg6xFhZBiVJkqZPv1zfqB5z3JKjsnQGTcj5DA3fVTrEWFkGJUmSps85ePN5TcA5y/d8/M25cd8cYdL9vl86wHhYBiVJkqZJt928Czi3dA71p6MXH7N+6Qwatx+UDjAelkFJkqTp5amimpDz81GPvmb5ln6Y0D9uA35VOsR4WAYlSZKml2VQE3bkkmPnZ7K8dA6Nyf8wNNxX/60sg5IkSdPrfOD20iHUny7Lxs5/yO1/WTqHxqSvrhcEy6AkSdK06raby4Eflc6h/nXkkmN3yGRJ6RwaVV9dLwiWQUmSpJngqaKasG5uvf1FuatHB3vbxQwNX186xHhZBiVJkqbfWYD3jdOEHb34mN0yuad0Dq3Wd0sHmIhJl8GImBMRv46IM+vnO0XEeRFxRUR8NSLm1dPXqZ9fWb/emOy6JUmS+kG33bwO+EnpHOpfN7LZVj9b/ri+GqlywJxeOsBETMWRwWOAy0c8/yDw0czclepi6cPr6YcDt2fmLsBH6/kkSZIGxRdKB1B/O3bJUX+TyZ2lc+ghfsvQ8GWlQ0zEpMpgRGwHNIHP188DeAbwjXqWU4Hn1d8fUj+nfn3/en5JkqRB8A3wND9N3O1stFln+YJfl86hh+jLo4Iw+SODHwPeBvff+2Rz4I7MXFo/XwhsW3+/LXAtQP36cD2/JEnSrNdtN+8CziidQ/3t7Uteu+fyjFtL59D9lgNfLh1ioiZcBiPiOcDNmXnhyMmrmDXH8NrI5R4RERdExAWLFi2aaDxJkqRe5KmimpS/sN6GX1n29EtK59D9fs7Q8MLSISZqMkcG/w54bkR0ga9QnR76MWCTiJhbz7MdsGKI1YXA9gD16xsDt6280Mw8MTP3ysy95s+fP4l4kiRJPecHwM2lQ6i/vWfpK/ZZlmvdUDqHgD4+RRQmUQYz8x2ZuV1mNoAXA+dk5suAHwMvqGc7DPhO/f0Z9XPq18/JTIdYliRJA6Pbbi6l+hBdmrD7mLfuicuaV5bOIRbzwFgpfWk67jP4duDNEXEl1TWBJ9XTTwI2r6e/GWhNw7olSZJ6naeKatL+c+mhC5bknGtK5xhwZzI0fHvpEJMxJWUwM3+Smc+pv78qM/fOzF0y89DMvK+efm/9fJf69aumYt2SJEn9pNtuXgD8vnQO9belzF37I0tfcG3pHAPuU6UDTNZ0HBmUJEnSmp1cOoD63wnLDt733lzb00XLuJih4XNKh5gsy6AkSdLM+xzwl9Ih1N+Stdb6t6Uv9zYTZXyidICpYBmUJEmaYd128w7glNI51P++tOyZ+9yd615WOseAuZU+H0V0BcugJElSGR9nFfdclsbr7UuOuKd0hgHzOYaGZ8U2twxKkiQV0G03rwDOLJ1D/a+zfMHf3pYb/qZ0jgGxlFkwcMwKlkFJkqRy/qN0AM0Oxyw5ek7pDAPiWwwNLywdYqpYBiVJkgrptps/A/6vdA71v58vf9zf3JCbnV86xwD4eOkAU8kyKEmSVNYHSgfQ7PD6xcdulOl1qNPofxganlUf3lgGJUmSCuq2m2cCvy2dQ/3vN7nLbn/Oh59bOscs9i+lA0w1y6AkSVJ57dIBNDscueS4h2eyrHSOWehMhobPKx1iqlkGJUmSyvsaHh3UFPhjbr/Tpdn4Zekcs0wC7y4dYjpYBiVJkgrrtpvLgbeUzqHZ4fVLjt0pk/tK55hFvsnQ8Ky8dYdlUJIkqQd0280fAZ3SOdT/rs0ttz0vH+21g1NjOfCvpUNMF8ugJElS7/hnqptaS5PypsVv2D2Tv5TOMQt8iaHhy0uHmC6WQUmSpB7RbTd/D5xYOof6381sOv9Hy/e8oHSOPrcYGCodYjpZBiVJknrLEHBn6RDqf29ecuQemQyXztHHPsbQ8J9Kh5hOlkFJkqQe0m03FwHvL51D/e9ONtj428uf9OvSOfrU9cB7S4eYbpZBSZKk3vMxoFs6hPrfvyx59ROXZywqnaMPvY2h4btLh5hulkFJkqQe02037wPeUTqH+t9fWfdhpy07YNYOgDJNfs7Q8OmlQ8wEy6AkSVIP6rabXwHOKZ1D/e/9S1+2z9Jc67rSOfrEEuDI0iFmimVQkiSpdx0OzPpT1TS9FrP2Op9adshVpXP0iQ8zNHxZ6RAzxTIoSZLUo7rtZhd4a+kc6n+fWPoP+y7OuX8unaPH/YkJDhoTESdHxM0RcckUZ5pWlkFJkqQe1m03TwDOLp1D/W0Zc+Z+cOmLbyydo4clcARDw/dO8OdPAQ6cujgzwzIoSZLU+16D9x7UJJ207KAFf815fyido0d9lKHhCV+jm5k/A26bwjwzwjIoSZLU47rt5jXAW0rnUL+L+Nelr/JDhYf6HfDO0iFKsAxKkiT1gW67+Xng+6VzqL99Y9lTn3hnrn9x6Rw95D7gZQwN31c6SAmWQUmSpP7xGuCO0iHU396y5MhlpTP0kHcwNNxXg75MJcugJElSn+i2m9cBx5bOof529vK99liUG19UOkcP+B/gY6VDlGQZlCRJ6iPddvNU4LTSOdTf3rD4jeuWzlDY7cArGRrOqVhYRHwZ+CWwW0QsjIjDp2K5021u6QCSJEkat9cBjwX2LB1E/em83H33a5dvcd72a92yT+kshbyWoeHrpmphmfmSqVrWTPLIoCRJUp/ptpv3As8HbimdRf3rqCXHbpbJ8tI5CvgAQ8PfLB2iF1gGJUmS+lB9u4kXAg4Gogm5OB+x6xW57bmlc8ywDvAvpUP0CsugJElSn+q2mz8G3l46h/rXkUuO2zaTJaVzzJDfAy9laHgQj4aukmVQkiSpj3Xbzf8Evlw6h/rTVbnNjr/JXQbh6OAwcAhDw3eWDtJLLIOSJEn97zXAb0uHUH86evGbdsnk3tI5ptFyqiOCfywdpNdYBiVJkvpct938K9WAMreVzqL+cz1bbP2L5Y89r3SOafQuhobPKh2iF1kGJUmSZoFuu/ln4B+B+0pnUf85ZsnRj83krtI5psEXGRpulw7RqyyDkiRJs0S33fwJ8DIYyNsFaBJuY+PNv7d874tK55hiZwKvKh2il1kGJUmSZpFuu/lN4KjSOdR/3rbkiCcsT24vnWOK/BQ4lKHhpaWD9DLLoCRJ0izTbTc/C/xr6RzqL3ez/kZfX/a035XOMQUuAp7L0PBsHhRnSlgGJUmSZqFuu/le4KOlc6i/HL/0sL2XZdxUOsck/AE40FtIjI1lUJIkaZbqtptvBj5TOof6x72ss95Jy579h9I5Juha4ACGhheVDtIvLIOSJEmz29HAf5UOof7xoaUv2ndJzrm2dI5xWkRVBPstd1GWQUmSpFms224m1U3pv1w6i/rDUuau/fGl/3BN6RzjcB3wVIaG+/WIZjGWQUmSpFmu224uB14OfL50FvWHTy07ZN97c+0/lc4xBn8CnsTQ8OWlg/Qjy6AkSdIA6Laby7rt5muBfy+dRb0vWWut9y39p16/9u4S4MkMDXdLB+lXlkFJkqQB0m033w28AW9Mr1F8YdkBC+7OdS8rnWM1zqc6NfSG0kH6mWVQkiRpwHTbzU8BLwbuK51Fve0dS17z19IZVuEnwP4MDd9WOki/swxKkiQNoG67+XXgIMD7sWm1vrt8v71uzw1+WzrHCGcCBzE0fFfpILOBZVCSJGlAddvNHwNPA/r5JuOaZsctOSpKZ6h9AngeQ8P3lg4yW1gGJUmSBli33fw1sB9wZeks6k0/Wb7H427MTS8oGGExcDhDw8cwNLysYI5ZZ8JlMCK2j4gfR8TlEXFpRBxTT98sIs6OiCvqr5vW0yMiPhERV0bE7yJiz6n6JSRJkjRx3XbzKmBf4Iels6g3HbX4mA0yyQKrvgl4OkPDJxdY96w3mSODS4G3ZOajgQXA0RGxO9ACfpSZuwI/qp9DdU76rvXjCOAzk1i3JEmSplC33byF6v3aEI40qpVclI981NW51XkzvVrgiQwN/98Mr3dgTLgMZuYNmXlR/f1dwOXAtsAhwKn1bKcCz6u/PwQ4LSvnAptExNYTTi5JkqQp1W03l3fbzfdQlcJbSudRbzlyyXFbZjJTp2l+lepm8tfO0PoG0pRcMxgRDeAJwHnAVpl5A1SFEdiynm1bYOR/zIX1tJWXdUREXBARFyxa1Ov3uZQkSZp9uu3mD6ne251bOot6x+9zh0dcljv+cppXsxR4B0PDL2Zo+J5pXtfAm3QZjIgNgG8Cx2bmmoYmXtUoRA857zgzT8zMvTJzr/nz5082niRJkiag224uBJ5CNYKjBMDrlxzbyGTxNC3+T1RHA9vTtHytZFJlMCLWpiqCp2fmt+rJN604/bP+enM9fSGw/Ygf3w64fjLrlyRJ0vTptptLuu3mMcALAe/rJq7JrbY7P3ebjqODpwB7MDQ809clDrTJjCYawEnA5Zn5kREvnQEcVn9/GPCdEdNfUY8qugAYXnE6qSRJknpXfYP6JwIlby+gHvHGxW98dCZ/naLF3Q68kKHhVzE0fPcULVNjNJkjg38HvBx4RkT8pn48G2gDB0TEFcAB9XOAs4CrqO5h8zngqEmsW5IkSTOo227+gWoE+bcCXss1wG5isy1/vHyP86dgUT8BHsfQ8NenYFmagLkT/cHM/AWrvg4QYP9VzJ/A0RNdnyRJksrqtpvLgP9otDrfpvpw/+mFI6mQ45Yc9fjfrHXEcAQbT+DH7wOOBz7M0LC3MSloSkYTlSRJ0uDotpt/ovrw/whguHAcFTDMBpucsXy/X0/gR38IPJah4Q9aBMuzDEqSJGncuu1mdtvNzwG788AYERog71xy+F7LM8Z6P8rrgRcxNPwshoavnM5cGjvLoCRJkias225e3203n0c14uhNpfNo5vyF9TY4fdn+l44y2zLg48CjGBr+2gzE0jhYBiVJkjRp9YijuwOfBJYUjqMZ8t6lL1+wNNda3e3izgX2Ymj4WIaGvTVJD7IMSpIkaUp0283buu3mm4DHAN8abX71v8Wsvc4Jyw7+00qTrwdeA+zH0PBvCsTSGFkGJUmSNKW67eYV3XbzH4EnUR0d0iz20aUv2HdxzrkauAN4B7ALQ8MnMTSchaNpFJZBSZIkTYtuu/m/3XZzX+AQ4Hel82h6LGPO4vcsPewEYGeGhtsMDXsfyj5hGZQkSdK06rabZwB7AC8B/lg4jqbOPcBHgJ3e976Pthkavq10II3PhG86L0mSJI1Vt91M4CuNVufrVKXwOGDPsqk0QfcAJwLtbrt5Y+kwmjjLoCRJkmZMt91cBnwR+GKj1XkScCzwPGBO0WAaiy7waeCkbrvpUcBZwDIoSZKkIrrt5i+AXzRanR2BN1CNQLlJ2VRaSQL/A/w/4Mxuu7m8cB5NIcugJEmSiuq2m1cDb220OkPAYcCbgN2KhtJdwKnAp7rt5u9Lh9H0sAxKkiSpJ3Tbzb8An260Op8BDgQOB5rAukWDDZbfA58CTu22m94ofpazDEqSJKmn1IPNfA/4XqPV2RB4LvBi4O+BeSWzzVJXAd8Avt5tNy8oHUYzxzIoSZKknlUfnTodOL3R6mwKPJ+qGD4DB52ZjCt5oABeVDqMyrAMSpIkqS90283bgZOBkxutzpbAP1IVwyfh/bPH4o/A14FvdNvN35QOo/Isg5IkSeo73XbzZuAzwGcarc7GwFOAp1MdMXwcEAXj9Yo7gF8APwO+3203Ly6cRz3GMihJA6rR6pSOMKpuu1k6gqQ+0G03h4Hv1g8arc7mwFOpyuHTgceUSzejbgZ+DvyUqgBe7K0gtCaWQUmSJM0q3XbzVuBb9YP6lNKnUZ1O+niqI4f9fj/DpBr45Tzq8uctIDRelkFJkiTNavUppV+rHwDUN7p/XP14NPAoqnsbblAi4xosA64GrgAuBS6pH5fVt+KQJswyKEmSpIFT3+j+aupTS1dotDrbUZXCbYAtV3psNeL7dSYZYSlwa/24ZaWvN1GN9nkFcFW33VwyyXVJq2QZlCRJkmrddnMhsHC0+RqtzkZUpXATqpFMYw1fVzzupi589XWOUlGWQUmSJGmcuu3mncCdpXNIk+H9WCRJkiRpAFkGJUmSJGkAWQYlSZIkaQBZBiVJkiRpAFkGJUmSJGkAWQYlSZIkaQBZBiVJkiRpAFkGJUmSJGkAWQYlSZIkaQBZBiVJkiRpAM0tHUD9p9HqlI6wRt12s3QESZIkqed5ZFCSJEmSBpBlUJIkSZIGkKeJSupLvX66MnjKsiRJ6m0eGZQkSZKkAWQZlCRJkqQB5GmiUgGe4ihJkqTSPDIoSZIkSQPIMihJkiRJA8gyKEmSJEkDyDIoSZIkSQPIMihJkiRJA8gyKEmSJEkDyDIoSZIkSQPIMihJkiRJA8gyKEmSJEkDyDIoSZIkSQNoxstgRBwYEX+IiCsjojXT65ckSZIkzXAZjIg5wKeAg4DdgZdExO4zmUGSJEmSNPNHBvcGrszMqzJzMfAV4JAZziBJkiRJA2+my+C2wLUjni+sp0mSJEmSZlBk5sytLOJQ4FmZ+Zr6+cuBvTPzjSPmOQI4on66G/CHGQs4MVsAt5QO0efchpPnNpw8t+HU6PXtuGNmzi8dQpKkXjB3hte3ENh+xPPtgOtHzpCZJwInzmSoyYiICzJzr9I5+pnbcPLchpPnNpwabkdJkvrHTJ8mej6wa0TsFBHzgBcDZ8xwBkmSJEkaeDN6ZDAzl0bEG4AfAHOAkzPz0pnMIEmSJEma+dNEycyzgLNmer3TqG9Oae1hbsPJcxtOnttwargdJUnqEzM6gIwkSZIkqTfM9DWDkiRJkqQeYBmUJEmSpAFkGZQkSZKkATTjA8hIABGxFbAtkMD1mXlT4Uh9KSI2AzIzby+dpV+5DSfHf8uSJPUvB5CZAN/8TFxE7AGcAGwMXFdP3g64AzgqMy8qla1fRMQOwIeA/am2WwAbAecArczslkvXH9yGk+e/ZUmS+p9lcBx88zN5EfEb4HWZed5K0xcAn83Mx5dJ1j8i4pfAx4BvZOayetoc4FDg2MxcUDJfP3AbTp7/liVJ6n+WwXHwzc/kRcQVmbnral67MjN3melM/WaUbbja1/QAt+Hk+W9ZkqT+5zWD4/OwlYsgQGaeGxEPKxGoD30vIjrAacC19bTtgVcA3y+Wqr9cGBGfBk7lwdvwMODXxVL1F7fh5PlvWZKkPueRwXGIiE8AO7PqNz9/zsw3lMrWTyLiIOAQqusuA1gInJGZZxUN1iciYh5wOA/ehtcC3wVOysz7CsbrC27DqeG/ZUmS+ptlcJx88yNJkiRpNrAMqmdExBGZeWLpHP0sIp6TmWeWztHP3IaT579lSZL6gzednyIRcUTpDLNAlA4wCzyxdIBZwG04ef5bliSpDziAzNTxzc8YRcSjqE6zPS8z7x7x0tWFIvWdiNib6kbp50fE7sCBwO8z8/jC0fpWRJyWma9wG05MRDwJ2Bu4JDM/WzqPJEkanWVw6iwuHaAfRMSbgKOBy4GTIuKYzPxO/fL7cRTCUUXE8cBBwNyIOBvYB/gJ0IqIJ2Tm+0rm6wcRccbKk4CnR8QmAJn53JlP1V8i4leZuXf9/Wup/l1/Gzg+IvbMzHbRgJIkaVReMzhFIuKazNyhdI5eFxEXA/tm5t0R0QC+AXwhMz8eEb/OzCcUDdgH6m24B7AOcCOwXWbeGRHrUR1tfVzRgH0gIi4CLgM+DyRVGfwy8GKAzPxpuXT9YeS/14g4H3h2Zi6qb7Nzbmb+TdmEkiRpNP+/vTtWrSIIwzD8foqNBIJV7PQSBK3FzkIbC9t4CVZWXoR34A2Y0guwshOCvWBhEbEQAiYgx/NZ5CyoENi1OIdl36dZmGn+athv/5kdO4MTJPl42RRwsM1aZuzqsDW07eckD4CjJLdwq+1Yq7a/gLMkn9qeArQ9T7LecW1zcQ94DrwEXrQ9TnJuCJzkSpIbXJw9T9tvAG1/JFnttjRJkjSGYXCaA+Ah8P2f8QDvt1/OLJ0kudP2GGDTIXwMvAbsJIzzM8n1tmfA3WEwyT5gGByh7Rp4leTN5vkV18Op9oEPXKx/TXKz7UmSPfywI0nSLPjyM81bYG8IMn9K8m775czSIfBX16DtCjhM4k8nxrk/XIq+CTWDa8Cz3ZQ0T22/AE+TPAJOd13PnLS9fcnUGniyxVIkSdJ/8sygJEmSJC2Q9wxKkiRJ0gIZBiVJkiRpgQyDkiRJkrRAhkFJkiRJWiDDoCRJkiQt0G9VDJxdFdA2iwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### LABEL DISTRIBUTION ###\n",
    "\n",
    "plt.figure(figsize=(16,6))\n",
    "\n",
    "plt.subplot(121)\n",
    "df.ride_rating.value_counts().sort_index().plot.bar(title='Ride Rating Distribution')\n",
    "\n",
    "plt.subplot(122)\n",
    "df.sentiment.value_counts().plot.pie(title='Sentiment Distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "### REMOVE 3 STARS REVIEWS ###\n",
    "\n",
    "df = df.loc[df.ride_rating!=3, :].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CLEAN TEXT (NOT REQUIRED) ###\n",
    "\n",
    "df['text'] = df.ride_review.str.lower().str.replace(r'['+string.digits+string.punctuation+']', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "### TRAIN TEST SPLIT ###\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(df['text'].values, df['sentiment'].values, \n",
    "                                                    random_state = 42, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(min_df=3,\n",
       "                stop_words={'a', 'about', 'above', 'after', 'again', 'against',\n",
       "                            'ain', 'all', 'am', 'an', 'and', 'any', 'are',\n",
       "                            'aren', \"aren't\", 'as', 'at', 'be', 'because',\n",
       "                            'been', 'before', 'being', 'below', 'between',\n",
       "                            'both', 'but', 'by', 'can', 'couldn', \"couldn't\", ...})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### CREATE FEATURES WITH TFIDF ###\n",
    "\n",
    "stop = set(stopwords.words('english'))\n",
    "tfidf = TfidfVectorizer(min_df=3, stop_words=stop)\n",
    "tfidf.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "### REVIEWS LENGTH DISTRIBUTION ON TRAIN ###\n",
    "\n",
    "c = Counter([len(s.split()) for s in X_train])\n",
    "lenghts = np.asarray(list(c.keys()))\n",
    "freq = np.asarray(list(c.values()))\n",
    "freq = freq/freq.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhgAAAFlCAYAAACp0JGSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAUVklEQVR4nO3df4xlZ3kf8O8Tr/lNYhsPaGtDFxKLgKKyplvX1FVEDKQGR+BIVMKKUv9hadMKVGhRkyWV0iC1kpESnFaKUJ2YYFWUQPlRI5smsYxRRFWZrsGYdRZqA1ticL1DwQFaicbm6R/3LAzLLDPeee/unrufj3R1z3nve899npl7R1+de86Z6u4AAIz0E6e7AABg9QgYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcLtO5YtdeOGFvWfPnlP5kgDAktxzzz1f7+61zR47pQFjz549OXjw4Kl8SQBgSarqf57oMV+RAADDCRgAwHDbDhhVdU5VfaaqbpvWn19Vd1fVA1X1/qp60vLKBADm5InswXhzksMb1t+R5MbuviTJN5NcP7IwAGC+thUwquriJFcn+cNpvZJcmeSD05RbklyzjAIBgPnZ7h6M30vy60m+N60/K8mj3f3YtP5Qkos2e2JV7a+qg1V1cH19fUfFAgDzsGXAqKpfSnK0u+/ZOLzJ1E3/73t339Td+7p739rapqfKAgArZjvXwbgiyWur6jVJnpLkJ7PYo3FeVe2a9mJcnORryysTAJiTLfdgdPfbuvvi7t6T5A1JPt7dv5LkriSvn6Zdl+TWpVUJAMzKTq6D8RtJ/nlVPZjFMRk3jykJAJi7J3Sp8O7+RJJPTMtfSnLZ+JIAgLlzJU8AYDgBAwAY7pT+N9W52nPg9qVs98gNVy9luwBwutmDAQAMJ2AAAMMJGADAcAIGADCcgAEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcLtOdwGj7Dlw++ku4QlbZs1Hbrh6adsGgK3YgwEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcAIGADDclgGjqp5SVZ+qqs9W1f1V9fZp/D1V9eWqune67V1+uQDAHGznv6l+N8mV3f2dqjo3ySer6r9Mj/2L7v7g8soDAOZoy4DR3Z3kO9PqudOtl1kUADBv29mDkao6J8k9SX4mye93991V9U+S/Juq+q0kdyY50N3f3eS5+5PsT5LnPe95wwrnx9tz4PalbfvIDVcvbdsArIZtHeTZ3Y93994kFye5rKp+Lsnbkvxskr+T5IIkv3GC597U3fu6e9/a2tqgsgGAM9kTOoukux9N8okkV3X3w73w3SR/lOSyJdQHAMzQds4iWauq86blpyZ5ZZLPV9XuaaySXJPk0DILBQDmYzvHYOxOcst0HMZPJPlAd99WVR+vqrUkleTeJP94iXUCADOynbNI7kty6SbjVy6lIgBg9lzJEwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACG2zJgVNVTqupTVfXZqrq/qt4+jT+/qu6uqgeq6v1V9aTllwsAzMF29mB8N8mV3f2SJHuTXFVVlyd5R5Ibu/uSJN9Mcv3yygQA5mTLgNEL35lWz51uneTKJB+cxm9Jcs1SKgQAZmdbx2BU1TlVdW+So0nuSPLFJI9292PTlIeSXLScEgGAudlWwOjux7t7b5KLk1yW5EWbTdvsuVW1v6oOVtXB9fX1k68UAJiNJ3QWSXc/muQTSS5Pcl5V7ZoeujjJ107wnJu6e19371tbW9tJrQDATGznLJK1qjpvWn5qklcmOZzkriSvn6Zdl+TWZRUJAMzLrq2nZHeSW6rqnCwCyQe6+7aq+oskf1xV/zrJZ5LcvMQ6AYAZ2TJgdPd9SS7dZPxLWRyPAQDwQ1zJEwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOG2DBhV9dyququqDlfV/VX15mn8t6vqq1V173R7zfLLBQDmYNc25jyW5K3d/emqemaSe6rqjumxG7v7d5ZXHgAwR1sGjO5+OMnD0/K3q+pwkouWXRgAMF9P6BiMqtqT5NIkd09Db6qq+6rq3VV1/uDaAICZ2nbAqKpnJPlQkrd097eSvCvJTyfZm8Uejt89wfP2V9XBqjq4vr4+oGQA4Ey3rYBRVedmES7e290fTpLufqS7H+/u7yX5gySXbfbc7r6pu/d19761tbVRdQMAZ7DtnEVSSW5Ocri737lhfPeGab+c5ND48gCAOdrOWSRXJPnVJJ+rqnunsd9Mcm1V7U3SSY4k+bWlVAgAzM52ziL5ZJLa5KGPjS8HAFgFruQJAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcAIGADCcgAEADCdgAADDCRgAwHDb+Wdn8EP2HLh9ads+csPVS9s2AKeOPRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcAIGADCcgAEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMt2XAqKrnVtVdVXW4qu6vqjdP4xdU1R1V9cB0f/7yywUA5mA7ezAeS/LW7n5RksuTvLGqXpzkQJI7u/uSJHdO6wAAWweM7n64uz89LX87yeEkFyV5XZJbpmm3JLlmWUUCAPPyhI7BqKo9SS5NcneS53T3w8kihCR59ujiAIB52nbAqKpnJPlQkrd097eewPP2V9XBqjq4vr5+MjUCADOzrYBRVedmES7e290fnoYfqard0+O7kxzd7LndfVN37+vufWtrayNqBgDOcNs5i6SS3JzkcHe/c8NDH01y3bR8XZJbx5cHAMzRrm3MuSLJryb5XFXdO439ZpIbknygqq5P8pUk/3A5JQIAc7NlwOjuTyapEzz8irHlAACrwJU8AYDhBAwAYLjtHIMBp8yeA7cvbdtHbrh6adsG4IfZgwEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcAIGADCcgAEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcAIGADCcgAEADCdgAADDCRgAwHACBgAw3JYBo6reXVVHq+rQhrHfrqqvVtW90+01yy0TAJiT7ezBeE+SqzYZv7G79063j40tCwCYsy0DRnf/eZJvnIJaAIAVsZNjMN5UVfdNX6Gcf6JJVbW/qg5W1cH19fUdvBwAMBcnGzDeleSnk+xN8nCS3z3RxO6+qbv3dfe+tbW1k3w5AGBOTipgdPcj3f14d38vyR8kuWxsWQDAnJ1UwKiq3RtWfznJoRPNBQDOPru2mlBV70vy8iQXVtVDSf5VkpdX1d4kneRIkl9bYo0AwMxsGTC6+9pNhm9eQi0AwIpwJU8AYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACG2zJgVNW7q+poVR3aMHZBVd1RVQ9M9+cvt0wAYE62swfjPUmuOm7sQJI7u/uSJHdO6wAASbYRMLr7z5N847jh1yW5ZVq+Jck1g+sCAGbsZI/BeE53P5wk0/2zTzSxqvZX1cGqOri+vn6SLwcAzMnSD/Ls7pu6e19371tbW1v2ywEAZ4CTDRiPVNXuJJnuj44rCQCYu5MNGB9Nct20fF2SW8eUAwCsgu2cpvq+JP8tyQur6qGquj7JDUleVVUPJHnVtA4AkCTZtdWE7r72BA+9YnAtAMCKcCVPAGA4AQMAGE7AAACGEzAAgOEEDABgOAEDABhOwAAAhhMwAIDhBAwAYDgBAwAYbstLhcOq2HPg9qVt+8gNVy9t2wBzZA8GADCcgAEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMMJGADAcAIGADCcgAEADCdgAADDCRgAwHACBgAwnIABAAwnYAAAwwkYAMBwu3by5Ko6kuTbSR5P8lh37xtRFAAwbzsKGJNf6O6vD9gOALAifEUCAAy304DRSf6squ6pqv2bTaiq/VV1sKoOrq+v7/DlAIA52GnAuKK7X5rk1UneWFU/f/yE7r6pu/d19761tbUdvhwAMAc7Chjd/bXp/miSjyS5bERRAMC8nXTAqKqnV9Uzjy0n+cUkh0YVBgDM107OInlOko9U1bHt/Mfu/pMhVQEAs3bSAaO7v5TkJQNrAQBWhNNUAYDhBAwAYLgRV/IEZmjPgduXtu0jN1y9tG0D82APBgAwnIABAAwnYAAAwwkYAMBwAgYAMJyAAQAMJ2AAAMO5DgacwZZ5rQqAZbIHAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOFcBwMGcL0KTpdlvveO3HD10rbN6rMHAwAYTsAAAIYTMACA4QQMAGA4AQMAGE7AAACGEzAAgOFcBwMYbq7XZphr3czfKr737MEAAIYTMACA4QQMAGC4HQWMqrqqqr5QVQ9W1YFRRQEA83bSAaOqzkny+0leneTFSa6tqhePKgwAmK+d7MG4LMmD3f2l7v5/Sf44yevGlAUAzNlOAsZFSf5yw/pD0xgAcJbbyXUwapOx/pFJVfuT7J9Wv1NVX9jBa57IhUm+voTtnmnOlj4Tva6qHfda7xhUyXL9SJ8zqfuHbLNm798z3Em+97bb69880QM7CRgPJXnuhvWLk3zt+EndfVOSm3bwOluqqoPdvW+Zr3EmOFv6TPS6qs6WXs+WPhO9rqoRve7kK5L/nuSSqnp+VT0pyRuSfHQnxQAAq+Gk92B092NV9aYkf5rknCTv7u77h1UGAMzWjv4XSXd/LMnHBtWyE0v9CuYMcrb0meh1VZ0tvZ4tfSZ6XVU77rW6f+S4TACAHXGpcABguFkHjFW7VHlVvbuqjlbVoQ1jF1TVHVX1wHR//jReVfXvpt7vq6qXnr7Kn5iqem5V3VVVh6vq/qp68zS+ir0+pao+VVWfnXp9+zT+/Kq6e+r1/dOB0qmqJ0/rD06P7zmd9Z+Mqjqnqj5TVbdN6yvZa1UdqarPVdW9VXVwGlvF9/B5VfXBqvr89Jl92Yr2+cLpd3ns9q2qessq9pokVfXPpr9Jh6rqfdPfqqGf1dkGjFrNS5W/J8lVx40dSHJnd1+S5M5pPVn0fcl025/kXaeoxhEeS/LW7n5RksuTvHH63a1ir99NcmV3vyTJ3iRXVdXlSd6R5Map128muX6af32Sb3b3zyS5cZo3N29OcnjD+ir3+gvdvXfD6Xyr+B7+t0n+pLt/NslLsvjdrlyf3f2F6Xe5N8nfTvJ/k3wkK9hrVV2U5J8m2dfdP5fFiRpvyOjPanfP8pbkZUn+dMP625K87XTXNaCvPUkObVj/QpLd0/LuJF+Ylv99kms3mze3W5Jbk7xq1XtN8rQkn07yd7O4gM2uafz77+Uszsp62bS8a5pXp7v2J9DjxVn8Eb4yyW1ZXJBvVXs9kuTC48ZW6j2c5CeTfPn438uq9blJ37+Y5L+uaq/5wZW4L5g+e7cl+QejP6uz3YORs+dS5c/p7oeTZLp/9jS+Ev1Pu9ouTXJ3VrTX6SuDe5McTXJHki8mebS7H5umbOzn+71Oj/9Vkmed2op35PeS/HqS703rz8rq9tpJ/qyq7qnFFYuT1XsPvyDJepI/mr72+sOqenpWr8/jvSHJ+6blleu1u7+a5HeSfCXJw1l89u7J4M/qnAPGti5VvsJm339VPSPJh5K8pbu/9eOmbjI2m167+/Fe7Ha9OIt/EviizaZN97Pttap+KcnR7r5n4/AmU2ff6+SK7n5pFrvK31hVP/9j5s61111JXprkXd19aZL/kx98RbCZufb5fdNxB69N8p+2mrrJ2Cx6nY4jeV2S5yf5G0mensX7+Hg7+qzOOWBs61LlK+CRqtqdJNP90Wl81v1X1blZhIv3dveHp+GV7PWY7n40ySeyOO7kvKo6dh2ajf18v9fp8Z9K8o1TW+lJuyLJa6vqSBb/XfnKLPZorGKv6e6vTfdHs/iu/rKs3nv4oSQPdffd0/oHswgcq9bnRq9O8unufmRaX8VeX5nky9293t1/neTDSf5eBn9W5xwwzpZLlX80yXXT8nVZHK9wbPwfTUcyX57kr47txjvTVVUluTnJ4e5+54aHVrHXtao6b1p+ahYf7MNJ7kry+mna8b0e+xm8PsnHe/ri80zX3W/r7ou7e08Wn8ePd/evZAV7raqnV9Uzjy1n8Z39oazYe7i7/1eSv6yqF05Dr0jyF1mxPo9zbX7w9Uiymr1+JcnlVfW06e/xsd/r2M/q6T7YZIcHqrwmyf/I4jvtf3m66xnQz/uy+D7sr7NIjNdn8T3XnUkemO4vmOZWFmfRfDHJ57I4Gvi097DNPv9+FrvX7kty73R7zYr2+reSfGbq9VCS35rGX5DkU0kezGJX7JOn8adM6w9Oj7/gdPdwkn2/PMltq9rr1NNnp9v9x/7+rOh7eG+Sg9N7+D8nOX8V+5zqf1qS/53kpzaMrWqvb0/y+env0n9I8uTRn1VX8gQAhpvzVyQAwBlKwAAAhhMwAIDhBAwAYDgBAwAYTsAAAIYTMACA4QQMAGC4/w+IQ0q4Y/yH9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PLOT REVIEWS LENGTH DISTRIBUTION ON TRAIN ###\n",
    "\n",
    "plt.figure(figsize=(9,6))\n",
    "plt.hist(lenghts, bins=20)\n",
    "np.set_printoptions(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "### DEFINE MARKOV CHAIN GENERATOR ###\n",
    "\n",
    "def build_chain(texts):\n",
    "    \n",
    "    index = 1\n",
    "    chain = {}\n",
    "    \n",
    "    for text in texts:\n",
    "        \n",
    "        text = text.split()\n",
    "        for word in text[index:]:\n",
    "            key = text[index-1]\n",
    "            if key in chain:\n",
    "                chain[key].append(word)\n",
    "            else:\n",
    "                chain[key] = [word]\n",
    "            index += 1\n",
    "        \n",
    "        index = 1\n",
    "    \n",
    "    return chain\n",
    "\n",
    "def create_sentence(chain, lenght, seed):\n",
    "        \n",
    "    start = random.choice(list(chain.keys()))\n",
    "    text = [start]\n",
    "\n",
    "    while len(text) < lenght:\n",
    "        try:\n",
    "            after = random.choice(chain[start])\n",
    "            start = after\n",
    "            text.append(after)\n",
    "        except: #end of the sentence\n",
    "            #text.append('.')\n",
    "            start = random.choice(list(chain.keys()))\n",
    "    \n",
    "    return ' '.join(text)\n",
    "\n",
    "def Generator(x_train, y_train, rep, concat=False, seed=33):\n",
    "    \n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    new_corpus, new_labels = [], []\n",
    "    \n",
    "    for i,lab in enumerate(np.unique(y_train)):\n",
    "\n",
    "        selected = x_train[y_train == lab]\n",
    "        chain = build_chain(selected)\n",
    "\n",
    "        sentences = []\n",
    "        for i in range(rep):\n",
    "            lenght = int(np.random.choice(lenghts, 1, p=freq))\n",
    "            sentences.append(create_sentence(chain, lenght, seed))\n",
    "\n",
    "        new_corpus.extend(sentences)\n",
    "        new_labels.extend([lab]*rep)\n",
    "    \n",
    "    if concat:\n",
    "        return np.asarray(list(x_train) + new_corpus), np.asarray(list(y_train) + new_labels)\n",
    "    \n",
    "    return np.asarray(new_corpus), np.asarray(new_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "### AUGMENT TRAIN DATA ###\n",
    "\n",
    "X_train_new, y_train_new = Generator(X_train, y_train, rep=300, concat=True, seed=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1195\n",
       "1     440\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(y_train_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CREATE FAKE TEST DATA ###\n",
    "\n",
    "X_test_new, y_test_new = Generator(X_train, y_train, rep=100, concat=False, seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    100\n",
       "1    100\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(y_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "### PERFORMANCE REPORT ###\n",
    "\n",
    "def report_performances(model, y_test, x_test):\n",
    "    \n",
    "    performance = {}\n",
    "    x_test = tfidf.transform(x_test)\n",
    "    \n",
    "    performance['auc'] = roc_auc_score(y_test, model.predict_proba(x_test)[:,1])\n",
    "    performance['precision'] = precision_score(y_test, model.predict(x_test), average='macro')\n",
    "    performance['recall'] = recall_score(y_test, model.predict(x_test), average='macro')\n",
    "    performance['f1'] = f1_score(y_test, model.predict(x_test), average='macro')\n",
    "    \n",
    "    return performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FIT ON ORIGINAL DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'hidden_layer_sizes': (256, 128, 64)}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn = GridSearchCV(\n",
    "    MLPClassifier(random_state=42), cv=3, n_jobs=-1,\n",
    "    param_grid={'hidden_layer_sizes': [(256, 128, 64), (256, 128), (256,)]}\n",
    ")\n",
    "nn.fit(tfidf.transform(X_train), y_train)\n",
    "\n",
    "nn.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_train = report_performances(nn, y_test, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_train_new = report_performances(nn, y_test_new, X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 10, 'solver': 'liblinear'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic = GridSearchCV(\n",
    "    LogisticRegression(random_state=42), cv=3, n_jobs=-1,\n",
    "    param_grid={'solver': ['liblinear', 'lbfgs'],\n",
    "                'C': [0.01, 1, 10]}\n",
    ")\n",
    "logistic.fit(tfidf.transform(X_train), y_train)\n",
    "\n",
    "logistic.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic_train = report_performances(logistic, y_test, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic_train_new = report_performances(logistic, y_test_new, X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': None, 'n_estimators': 100}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = GridSearchCV(\n",
    "    RandomForestClassifier(random_state=42), cv=3, n_jobs=-1,\n",
    "    param_grid={'n_estimators': [50, 100, 200],\n",
    "                'max_depth': [None, 10, 20]}\n",
    ")\n",
    "rf.fit(tfidf.transform(X_train), y_train)\n",
    "\n",
    "rf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_train = report_performances(rf, y_test, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_train_new = report_performances(rf, y_test_new, X_test_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FIT ON ORIGINAL + GENERATED DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'hidden_layer_sizes': (256, 128, 64)}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn = GridSearchCV(\n",
    "    MLPClassifier(random_state=42), cv=3, n_jobs=-1,\n",
    "    param_grid={'hidden_layer_sizes': [(256, 128, 64), (256, 128), (256,)]}\n",
    ")\n",
    "nn.fit(tfidf.transform(X_train_new), y_train_new)\n",
    "\n",
    "nn.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_train_aug = report_performances(nn, y_test, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_train_aug_new = report_performances(nn, y_test_new, X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 10, 'solver': 'lbfgs'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic = GridSearchCV(\n",
    "    LogisticRegression(random_state=42), cv=3, n_jobs=-1,\n",
    "    param_grid={'solver': ['liblinear', 'lbfgs'],\n",
    "                'C': [0.01, 1, 10]}\n",
    ")\n",
    "logistic.fit(tfidf.transform(X_train_new), y_train_new)\n",
    "\n",
    "logistic.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic_train_aug = report_performances(logistic, y_test, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic_train_aug_new = report_performances(logistic, y_test_new, X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': None, 'n_estimators': 200}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = GridSearchCV(\n",
    "    RandomForestClassifier(random_state=42), cv=3, n_jobs=-1,\n",
    "    param_grid={'n_estimators': [50, 100, 200],\n",
    "                'max_depth': [None, 10, 20]}\n",
    ")\n",
    "rf.fit(tfidf.transform(X_train_new), y_train_new)\n",
    "\n",
    "rf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_train_aug = report_performances(rf, y_test, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_train_aug_new = report_performances(rf, y_test_new, X_test_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PERFORMANCE COMPARISON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAE/CAYAAACdNPbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5jV1X3v8fdXMJAESlWmNAHNkESDGIgi4eoAKljTI4mYUOU0ETWNh5ygDRZN9DSIlx4rRU2MekgqclEbY6rmaB68AAriheCAYxTwggbrVGqMARQVh9HVP2bPuB0G2MjGmQXv1/PM4++yfuv33Rl+mfnMWmvvSCkhSZIkSTnZp7ULkCRJkqSdZZCRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJJ2u4iYGhE3tXYduyoiKiMiRUT71q5FkvZ2BhlJ2gNFxNqIeCUiPll07O8iYlErltWiiBhRCAfXNjv+UEScVmIfKSI+v1sKlCS1SQYZSdpztQf+fnffpEyjE28Cp0ZEZRn62i0chZGktsUgI0l7rn8BJkfEn7d0MiJ6RcT8iPhTRDwTEX9TdG5RRPxd0f5pEfFQ0X6KiO9FxHPAc4VjP4mIlyLi9YhYHhFVO1HrBmA2cOG2GkTEGRGxOiLWR8S9EfGZwvEHC02eiIhNEXFyRCyOiK8Xzh9VqPevC/sjI6KmsL1PRPxjRLwYEX+IiLkR0aVwrnEa2bcj4j+A+1uo6euF0a8v7sRrlSSVgUFGkvZc1cAiYHLzE4UpZ/OBfwP+AhgHXBcRh+1E/ycCA4Hehf3HgMOB/Qv9/ioiOu5Ef/8EfD0ivtBCvScCFwAnARXAEuAXACmlYYVmX0opdUop/RJYDIwoHB8GvAAML9pfXNg+rfB1NPBZoBNwTbPbDwcOBf6qWU2nA5cDI1NKT+3E65QklYFBRpL2bFOAsyKiotnxE4C1KaVZKaX6lNIK4DbgGzvR92UppT+llN4GSCndlFJ6rdDfFUAHYKtQsi0ppf8CZgAXt3D6fxXutzqlVA/8X+DwxlGZFizmg8HlsqL94bwfZP4WuDKl9EJKaRNwPnBKs2lkU1NKbza+zoLvA+cCI1JKa0p9jZKk8jHISNIerDBS8Bvgh81OfQYYGBEbGr9o+KX+L3ei+5eKdyLiHwpTvzYW+usCdN3Jki8H/ioivtRCvT8pqvVPQADdt9HPo8AhEdGNhlGiucCBEdEVGAA0Tkf7NPBi0XUv0rC2qNu2XmfBucC1KaXakl+ZJKmsXLgoSXu+C4EVwBVFx14CFqeURm3jmjeBTxTttxRwUuNGYT3MD4BjgZUppfciYj0NYaNkKaXXIuLHwCXNTr0E/FNK6eYS+3krIpbT8GYHT6WU6iLiEeAc4PmU0h8LTV+mISQ1OgioB14BejR/nUWOA+6JiP9KKd1WSk2SpPJyREaS9nCFqU+/BM4uOvwbGkYsvhUR+xa+vhwRhxbO1wAnRcQnCm9r/O0d3KYzDQHgVaB9REwB/uxDlnwlMISGdSmNZgDnN67hiYguETG26PwrNKxxKbYYmMj708gWNduHhnU2kyKiZ0R0omHK2i8L09e2ZyVwPHBtRHy11BcmSSofg4wk7R0uBpo+Uyal9AYNowqn0DAq8V80TOvqUGhyFVBHQ0CYA+xoJORe4G7gWRqmZ22m5SlZO5RSeh2YRsObBjQeu6NQ3y0R8TrwFPCVosumAnMKU88a331tMQ0B68Ft7APcANxYOPb7Qt1nlVjnEzSsNfrXiPjKjtpLksorUmppxFySJEmS2i5HZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2Wu0DMbt27ZoqKytb6/aSJEmS2rjly5f/MaVU0dK5VgsylZWVVFdXt9btJUmSJLVxEfHits45tUySJElSdgwykiRJkrJjkJEkSZKUnVZbIyNJkiS1ZMuWLdTW1rJ58+bWLkUfkY4dO9KjRw/23Xffkq8xyEiSJKlNqa2tpXPnzlRWVhIRrV2OdrOUEq+99hq1tbX07Nmz5OucWiZJkqQ2ZfPmzRxwwAGGmL1ERHDAAQfs9AicQUaSJEltjiFm7/Jhvt8GGUmSJOlDuOyyy7j55ptbu4ydtmjRIh555JGdbjdjxgzmzp27O0vbKa6RkSRJUpt21fxny9rfpFGHlKWf++67j1tvvbUsfX2UFi1aRKdOnRgyZMhOtZswYcJHUV7JHJGRJEmSikybNo2rr74agEmTJnHMMccAsHDhQr75zW8C8Prrr1NXV0dFRcUHrl22bBlDhgzhiCOOYMiQITzzzDMAzJ49m4kTJza1O+GEE1i0aBEAM2fO5JBDDmHEiBF85zvfaWp32mmn8d3vfpejjz6az372syxevJgzzjiDQw89lNNOO62pr/vuu4/BgwfTr18/xo4dy6ZNmwCorKzkwgsvpF+/fvTp04enn36atWvXMmPGDK666ioOP/xwlixZwl133cXAgQM54ogjGDlyJK+88kqL7aZOncr06dMBqKmpYdCgQfTt25cxY8awfv16AEaMGMEPfvADBgwYwCGHHMKSJUvK+a35AIOMJEmSVGTYsGFNv4BXV1ezadMmtmzZwkMPPURVVRUACxYs4Nhjj93q2l69evHggw/y+OOPc/HFF3PBBRds914vv/wyl1xyCUuXLmX+/Pk8/fTTHzi/fv167r//fq666ipGjx7NpEmTWLlyJU8++SQ1NTX88Y9/5NJLL2XBggWsWLGC/v37c+WVVzZd37VrV1asWMF3v/tdpk+fTmVlJRMmTGDSpEnU1NRQVVXFUUcdxdKlS3n88cc55ZRTmDZtWovtip166qlcfvnl/O53v6NPnz5cdNFFTefq6+tZtmwZP/7xjz9wvNycWlZQ7iHL3aFcw6CSJEnatiOPPJLly5fzxhtv0KFDB/r160d1dTVLlixpGqm55557OP3007e6duPGjYwfP57nnnuOiGDLli3bvdeyZcsYPnw4+++/PwBjx47l2Wff/7109OjRRAR9+vShW7du9OnTB4DDDjuMtWvXUltby6pVqxg6dCgAdXV1DB48uOn6k046qek13X777S3WUFtby8knn8y6deuoq6vb4Vsgb9y4kQ0bNjB8+HAAxo8fz9ixY1u859q1a7fb165wREaSJEkqsu+++1JZWcmsWbMYMmQIVVVVPPDAAzz//PMceuihQEMAGTBgwFbX/uhHP+Loo4/mqaee4q677mp6S+H27dvz3nvvNbVrPJ5S2m4tHTp0AGCfffZp2m7cr6+vJ6XEqFGjqKmpoaamhlWrVjFz5sytrm/Xrh319fUt3uOss85i4sSJPPnkk/zsZz/b5Q8iLeWe5eCITE4euKy1KyjN0ee3dgWSJEm7ZNiwYUyfPp0bbriBPn36cM4553DkkUcSEaxcuZJevXrRrl27ra7buHEj3bt3BxrWxTSqrKzkuuuu47333uM///M/WbZsGQADBgxg0qRJrF+/ns6dO3Pbbbc1jbqUYtCgQXzve99jzZo1fP7zn+ett96itraWQw7Z9kyezp078/rrr7dY85w5c7bZrlGXLl3Yb7/9WLJkCVVVVdx4441NozMfJUdkJEmSpGaqqqpYt24dgwcPplu3bnTs2LFpncjdd9/N8ccf3+J15513Hueffz5Dhw7l3XffbTo+dOhQevbsSZ8+fZg8eTL9+vUDoHv37lxwwQUMHDiQkSNH0rt3b7p06VJynRUVFcyePZtx48bRt29fBg0atNU6m+ZGjx7NHXfc8YFF/GPHjqWqqoquXbtus12xOXPmcO6559K3b19qamqYMmVKyTWXS+xoOGt36d+/f6qurm6Ve7ckizUy7W9r7RJK44iMJEnaBatXr26awtUWjRo1irlz5/KpT32qLP1t2rSJTp06UV9fz5gxYzjjjDMYM2ZMWfrOSUvf94hYnlLq31J7p5ZJkiRJO2H+/Pll7W/q1KksWLCAzZs3c9xxx3HiiSeWtf89lUFGkiRJakWNn82ineMaGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkj6Eyy67jJtvvrm1yyibRYsW8cgjj+x0uxkzZjB37tzdWVqLfNcySZIktW0PXFbe/sr0mXf33Xcft956a1n6agsWLVpEp06dGDJkyE61mzBhwkdR3lYMMpL2HuX+Qbi7+KGyktSqpk2bRseOHTn77LOZNGkSTzzxBPfffz8LFy5k1qxZ3HTTTbz++uvU1dVRUVHBiy++yBlnnMGrr75KRUUFs2bN4qCDDvpAn8uWLeP73/8+b7/9Nh//+MeZNWsWX/jCF5g9ezbV1dVcc801AJxwwglMnjyZESNGMHPmTC6//HI+/elPc/DBB9OhQweuueYaTjvtND7+8Y/z9NNP8+KLLzJr1izmzJnDo48+ysCBA5k9ezbQELQuvPBC3nnnHT73uc8xa9YsOnXqRGVlJePHj+euu+5iy5Yt/OpXv6Jjx47MmDGDdu3acdNNN/HTn/6UDRs2cOmll1JXV8cBBxzAzTffzNtvv71Vu4ULF9KpUycmT55MTU0NEyZM4K233uJzn/scN9xwA/vttx8jRoxg4MCBPPDAA2zYsIGZM2dSVVW1S98ng4ykXXbV/Gdbu4SSTPL/8SRJJRg2bBhXXHEFZ599NtXV1bzzzjts2bKFhx56qOmX7wULFnDssccCMHHiRE499VTGjx/PDTfcwNlnn82vf/3rD/TZq1cvHnzwQdq3b8+CBQu44IILuO2227ZZw8svv8wll1zCihUr6Ny5M8cccwxf+tKXms6vX7+e+++/nzvvvJPRo0fz8MMPc/311/PlL3+ZmpoaevTowaWXXsqCBQv45Cc/yeWXX86VV17JlClTAOjatSsrVqzguuuuY/r06Vx//fVMmDChKZA03mPp0qVEBNdffz3Tpk3jiiuu2KrdwoULm+o69dRT+elPf8rw4cOZMmUKF110ET/+8Y8BqK+vZ9myZcybN4+LLrqIBQsW7NL3yR/rkiRJUpEjjzyS5cuX88Ybb9ChQwf69etHdXU1S5Ys4eqrrwbgnnvu4fTTTwfg0Ucf5fbbbwfgW9/6Fuedd95WfW7cuJHx48fz3HPPERFs2bJluzUsW7aM4cOHs//++wMwduxYnn32/T8cjh49moigT58+dOvWjT59+gBw2GGHsXbtWmpra1m1ahVDhw4FoK6ujsGDBzddf9JJJzW91sbam6utreXkk09m3bp11NXV0bNnz+3WvHHjRjZs2MDw4cMBGD9+PGPHjm3xnmvXrt1uX6Vwsb8kSZJUZN9996WyspJZs2YxZMgQqqqqeOCBB3j++ec59NBDgYagMWDAgBavj4itjv3oRz/i6KOP5qmnnuKuu+5i8+bNALRv35733nuvqV3j8ZTSdmvs0KEDAPvss0/TduN+fX09KSVGjRpFTU0NNTU1rFq1ipkzZ251fbt27aivr2/xHmeddRYTJ07kySef5Gc/+1lTbR9WKffcGQYZSZIkqZlhw4Yxffp0hg0bRlVVFTNmzODwww8nIli5ciW9evWiXbt2AAwZMoRbbrkFgJtvvpmjjjpqq/42btxI9+7dAZrWsABUVlZSU1PDe++9x0svvcSyZcsAGDBgAIsXL2b9+vXU19dvdxpaSwYNGsTDDz/MmjVrAHjrrbc+MKLTks6dO/PGG2+0WPOcOXO22a5Rly5d2G+//ViyZAkAN954Y9PozO5gkJEkSZKaqaqqYt26dQwePJhu3brRsWPHpvUxd999N8cff3xT26uvvppZs2bRt29fbrzxRn7yk59s1d95553H+eefz9ChQ3n33Xebjg8dOpSePXvSp08fJk+eTL9+/QDo3r07F1xwAQMHDmTkyJH07t2bLl26lFx/RUUFs2fPZty4cfTt25dBgwbx9NNPb/ea0aNHc8cdd3D44YezZMkSpk6dytixY6mqqqJr167bbFdszpw5nHvuufTt25eampqmNTm7Q+xo2Gp36d+/f6qurm6Ve7ckh8XKk9rvXBJvNb7j0l4nh+cHfIYkKRerV69umsLVFo0aNYq5c+fyqU99arfeZ9OmTXTq1In6+nrGjBnDGWecwZgxY3brPVtTS9/3iFieUurfUnsX+0uSJEk7Yf78+R/JfaZOncqCBQvYvHkzxx13HCeeeOJHct9cGGQkSZKkNmj69OmtXUKb5hoZSZIkSdkxyEiSJKnNaa113GodH+b7bZCRJElSm9KxY0dee+01w8xeIqXEa6+9RseOHXfqOtfISJIkqU3p0aMHtbW1vPrqq61dij4iHTt2pEePHjt1jUFGkiRJbcq+++5Lz549W7sMtXFOLZMkSZKUnZJGZCLieOAnQDvg+pTSPzc7fxAwB/jzQpsfppTmlblWSZIkKV8PXNbaFZQmkw9m3uGITES0A64FvgL0BsZFRO9mzf4RuDWldARwCnBduQuVJEmSpEalTC0bAKxJKb2QUqoDbgG+1qxNAv6ssN0FeLl8JUqSJEnSB5USZLoDLxXt1xaOFZsKfDMiaoF5wFktdRQRZ0ZEdURU+y4UkiRJkj6sUoJMtHCs+Zt6jwNmp5R6AH8N3BgRW/WdUvp5Sql/Sql/RUXFzlcrSZIkSZQWZGqBA4v2e7D11LFvA7cCpJQeBToCXctRoCRJkiQ1V0qQeQw4OCJ6RsTHaFjMf2ezNv8BHAsQEYfSEGScOyZJkiRpt9hhkEkp1QMTgXuB1TS8O9nKiLg4Ir5aaPYPwHci4gngF8BpKaXm088kSZIkqSxK+hyZwmfCzGt2bErR9ipgaHlLkyRJknbsqvnPtnYJJZlU0m/eKlUpU8skSZIkqU0xF0qSpNL4qeSS2hBHZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7PiBmJIktbKr5j/b2iWUZJK/NUhqQxyRkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdgwykiRJkrJjkJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7JQWZiDg+Ip6JiDUR8cNttPmbiFgVESsj4t/KW6YkSZIkva/9jhpERDvgWmAUUAs8FhF3ppRWFbU5GDgfGJpSWh8Rf7G7CpYkSZKkUkZkBgBrUkovpJTqgFuArzVr8x3g2pTSeoCU0h/KW6YkSZIkva+UINMdeKlov7ZwrNghwCER8XBELI2I48tVoCRJkiQ1t8OpZUC0cCy10M/BwAigB7AkIr6YUtrwgY4izgTOBDjooIN2ulhJkiRJgtJGZGqBA4v2ewAvt9Dm/6eUtqSUfg88Q0Ow+YCU0s9TSv1TSv0rKio+bM2SJEmS9nKlBJnHgIMjomdEfAw4BbizWZtfA0cDRERXGqaavVDOQiVJkiSp0Q6DTEqpHpgI3AusBm5NKa2MiIsj4quFZvcCr0XEKuAB4NyU0mu7q2hJkiRJe7dS1siQUpoHzGt2bErRdgLOKXxJkiRJ0m5V0gdiSpIkSVJbYpCRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdgwykiRJkrJjkJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdgwykiRJkrJjkJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnZKCjIRcXxEPBMRayLih9tp942ISBHRv3wlSpIkSdIH7TDIREQ74FrgK0BvYFxE9G6hXWfgbOC35S5SkiRJkoqVMiIzAFiTUnohpVQH3AJ8rYV2lwDTgM1lrE+SJEmStlJKkOkOvFS0X1s41iQijgAOTCn9poy1SZIkSVKLSgky0cKx1HQyYh/gKuAfdthRxJkRUR0R1a+++mrpVUqSJElSkVKCTC1wYNF+D+Dlov3OwBeBRRGxFhgE3NnSgv+U0s9TSv1TSv0rKio+fNWSJEmS9mqlBJnHgIMjomdEfAw4Bbiz8WRKaWNKqWtKqTKlVAksBb6aUqreLRVLkiRJ2uvtMMiklOqBicC9wGrg1pTSyoi4OCK+ursLlCRJkqTm2pfSKKU0D5jX7NiUbbQdsetlSZIkSdK2lfSBmJIkSZLUlhhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdgwykiRJkrJjkJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdgwykiRJkrJjkJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpSdkoJMRBwfEc9ExJqI+GEL58+JiFUR8buIWBgRnyl/qZIkSZLUYIdBJiLaAdcCXwF6A+MionezZo8D/VNKfYF/B6aVu1BJkiRJalTKiMwAYE1K6YWUUh1wC/C14gYppQdSSm8VdpcCPcpbpiRJkiS9r5Qg0x14qWi/tnBsW74N3L0rRUmSJEnS9rQvoU20cCy12DDim0B/YPg2zp8JnAlw0EEHlViiJEmSJH1QKSMytcCBRfs9gJebN4qIkcD/Ab6aUnqnpY5SSj9PKfVPKfWvqKj4MPVKkiRJUklB5jHg4IjoGREfA04B7ixuEBFHAD+jIcT8ofxlSpIkSdL7dhhkUkr1wETgXmA1cGtKaWVEXBwRXy00+xegE/CriKiJiDu30Z0kSZIk7bJS1siQUpoHzGt2bErR9sgy1yVJkiRJ21TSB2JKkiRJUltikJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdgwykiRJkrJjkJEkSZKUHYOMJEmSpOwYZCRJkiRlxyAjSZIkKTsGGUmSJEnZMchIkiRJyo5BRpIkSVJ2DDKSJEmSsmOQkSRJkpQdg4wkSZKk7BhkJEmSJGXHICNJkiQpOwYZSZIkSdkxyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZccgI0mSJCk7BhlJkiRJ2THISJIkScqOQUaSJElSdkoKMhFxfEQ8ExFrIuKHLZzvEBG/LJz/bURUlrtQSZIkSWq0wyATEe2Aa4GvAL2BcRHRu1mzbwPrU0qfB64CLi93oZIkSZLUqJQRmQHAmpTSCymlOuAW4GvN2nwNmFPY/nfg2IiI8pUpSZIkSe8rJch0B14q2q8tHGuxTUqpHtgIHFCOAiVJkiSpuUgpbb9BxFjgr1JKf1fY/xYwIKV0VlGblYU2tYX95wttXmvW15nAmYXdLwDPlOuF7CW6An9s7SKkjPkMSbvGZ0jaNT5DO+8zKaWKlk60L+HiWuDAov0ewMvbaFMbEe2BLsCfmneUUvo58PNSKtbWIqI6pdS/teuQcuUzJO0anyFp1/gMlVcpU8seAw6OiJ4R8THgFODOZm3uBMYXtr8B3J92NNQjSZIkSR/SDkdkUkr1ETERuBdoB9yQUloZERcD1SmlO4GZwI0RsYaGkZhTdmfRkiRJkvZupUwtI6U0D5jX7NiUou3NwNjylqYWOC1P2jU+Q9Ku8RmSdo3PUBntcLG/JEmSJLU1payRkSRJkqQ2xSAjaa8REY/s4Py8iPjzj6oeaU8TEZUR8VRhe0RE/Ka1a5JaU0ScHRGrI+K2iHg0It6JiMmtXdeeoqQ1MpLU1kREu5TSuztzTUppyA7O//WuVSXlKSKChunm77V2LdIe5n8DXwHeBD4DnNi65exZHJFpYyLi1xGxPCJWFj5AlIjYVHT+GxExu7DdLSLuiIgnCl/b/SVNykXhr7pPR8SciPhdRPx7RHwiItZGxJSIeAgYGxGfi4h7Cs/MkojoVbi+xWej8VmKiE9FxIMRURMRT0VEVeH42ojoWtg+p3DuqYj4flFdqyPiXwvP6H0R8fFW+R9J2kVF/56vA1YA3yr8xXhFRPwqIjoV2n05Ih4pPEvLIqJz4dolhbYr/PkjbS0iZgCfpeFjSv42pfQYsKV1q9qzOCLT9pyRUvpT4ZejxyLitu20vRpYnFIaExHtgE4fTYnSR+ILwLdTSg9HxA00/FULYHNK6SiAiFgITEgpPRcRA87AipIAAAL+SURBVIHrgGPY8bPxP4F7U0r/VDj/ieKTEXEkcDowEAjgtxGxGFgPHAyMSyl9JyJuBb4O3FT2Vy99NL5Aw7/1KcDtwMiU0psR8QPgnIj4Z+CXwMkppcci4s+At4E/AKNSSpsj4mDgF4Af8icVSSlNiIjjgaNTSn9s7Xr2RAaZtufsiBhT2D6Qhl+atuUY4FSAwhSbjbu5Numj9FJK6eHC9k3A2YXtXwIU/lo8BPhVw6wYADoU/rujZ+Mx4IaI2Bf4dUqpptn5o4A7UkpvFu51O1BFw1/Vfl/UfjlQuQuvUWptL6aUlkbECUBv4OHC8/Qx4FEags66wl+SSSm9DhARnwSuiYjDgXeBQ1qjeEl7N4NMGxIRI4CRwOCU0lsRsQjoCBS/R3bHVihNag3N3xu+cf/Nwn/3ATaklA7f6Y5TejAihgH/g4YP8/2XlNLcoiaxjUsB3inafhdwaply1vg8BTA/pTSu+GRE9GXrZxFgEvAK8CUansXNu7NISWqJa2Tali7A+kKI6QUMKhx/JSIOjYh9gDFF7RcC34WGhc+FIX9pT3FQRAwubI8DHio+WfjL8O8jYiw0LFaOiC8VTm/32YiIzwB/SCn9KzAT6Nfs3g8CJxbW5XyShuduSflemtTmLAWGRsTnAQr/9g8BngY+HRFfLhzvHBHtafh5ta7w5gDfAtq1Ut2S9mIGmbblHqB9RPwOuISGHywAPwR+A9wPrCtq//fA0RHxJA1TXA77CGuVdrfVwPjC87A/8P9aaPO3wLcj4glgJfC1wvEdPRsjgJqIeJyGNS4/KT6ZUloBzAaWAb8Frk8pPV6G1yS1SSmlV4HTgF8UnrmlQK+UUh1wMvDTwnM2n4aZAdfR8HwupWFa2ZstdiwJgIj4y4ioBc4B/jEiav0D9K6LlFoaMZak1hMRlcBvUkpfbOVSJElSG+WIjCRJkqTsOCIjSZIkKTuOyEiSJEnKjkFGkiRJUnYMMpIkSZKyY5CRJEmSlB2DjCRJkqTsGGQkSZIkZee/AURuJvmMfQq4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1008x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PERFORMANCE ON THE STANDARD TEST ###\n",
    "\n",
    "plt.figure(figsize=(14,5))\n",
    "plt.bar(np.arange(len(nn_train_aug))-0.1, nn_train_aug.values(), \n",
    "        alpha=0.5, width=0.2, label='w/ augmentation')\n",
    "plt.bar(np.arange(len(nn_train))+0.1, nn_train.values(), \n",
    "        alpha=0.5, width=0.2, label='w/o augmentation')\n",
    "plt.xticks(range(len(nn_train_aug)), nn_train_aug.keys())\n",
    "plt.title('Neural Network'); plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAE/CAYAAACdNPbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5hU1Znv8e8LKJyIISqM44Da5EQUDAQQubeAgtGMeItEmBg1OnowQScYHS+TwWvGaIgYJzpoogE0iZoYM+ox3gVRRK7tBfGCiqGPxCgiFx2E1nX+qKItm4KuhoLuDd/P8/RD7bXXXvstmy3967XXrkgpIUmSJElZ0qyxC5AkSZKkhjLISJIkScocg4wkSZKkzDHISJIkScocg4wkSZKkzDHISJIkScocg4wkaauLiEsj4vbGrmNLRURFRKSIaNHYtUjSjs4gI0nboYhYHBHvRMQuBW3/HBFTG7GsoiJicD4c3FCn/amIOLXEMVJEfGWrFChJapIMMpK0/WoB/MvWPkmZZic+BE6OiIoyjLVVOAsjSU2LQUaStl8/Bc6LiC8V2xkRB0TEIxHxfkS8EhHfKtg3NSL+uWD71Ih4qmA7RcT3I+I14LV8288jYklErIyIuRFR2YBaPwAmAZdsrENEnBYRCyNieUQ8FBH75tufzHd5LiJWR8SJETEtIr6Z3z8wX+838ttDI6Iq/7pZRPwoIt6KiL9FxJSIaJPft/42stMj4i/A40Vq+mZ+9uurDXivkqQyMMhI0vZrDjAVOK/ujvwtZ48AvwX+DhgF3BgRBzZg/GOBPkCX/PZsoDuwe37c30dEqwaM92PgmxGxf5F6jwUuBo4H2gHTgd8BpJQOyXf7WkqpdUrpTmAaMDjffgjwBjCoYHta/vWp+a8hwJeB1sAv6px+ENAZ+Hqdmr4LXA0MTSm92ID3KUkqA4OMJG3fxgFnR0S7Ou1HAYtTSr9OKdWklOYBdwMnNGDsq1JK76eU/gcgpXR7SmlZfryfAS2BDULJxqSU/gpMBC4vsvv/5M+3MKVUA/wH0H39rEwR0/h8cLmqYHsQnwWZbwPXppTeSCmtBi4CRta5jezSlNKH699n3g+A84HBKaVFpb5HSVL5GGQkaTuWnym4H7iwzq59gT4R8cH6L3I/1P99A4ZfUrgRET/M3/q1Ij9eG6BtA0u+Gvh6RHytSL0/L6j1fSCA9hsZ5xmgU0TsSW6WaAqwd0S0BXoD629H+wfgrYLj3iK3tmjPjb3PvPOBG1JK1SW/M0lSWblwUZK2f5cA84CfFbQtAaallIZt5JgPgS8UbBcLOGn9i/x6mAuAw4AFKaVPI2I5ubBRspTSsoi4Driizq4lwI9TSr8pcZyPImIuuYcdvJhSWhsRM4BzgddTSu/lu75NLiSttw9QA7wDdKj7PgscDjwYEX9NKd1dSk2SpPJyRkaStnP5W5/uBM4paL6f3IzFdyJip/zXwRHROb+/Cjg+Ir6Qf6zx6fWcZldyAeBdoEVEjAO+uJklXwv0J7cuZb2JwEXr1/BERJuIGFGw/x1ya1wKTQPG8NltZFPrbENunc3YiOgYEa3J3bJ2Z/72tU1ZABwB3BARR5f6xiRJ5WOQkaQdw+VA7WfKpJRWkZtVGEluVuKv5G7rapnvMgFYSy4gTAbqmwl5CPgz8Cq527PWUPyWrHqllFYC15B7aMD6tnvy9d0RESuBF4EjCw67FJicv/Vs/dPXppELWE9uZBvgVuC2fNub+brPLrHO58itNfplRBxZX39JUnlFSsVmzCVJkiSp6XJGRpIkSVLmGGQkSZIkZY5BRpIkSVLmGGQkSZIkZY5BRpIkSVLmNNoHYrZt2zZVVFQ01uklSZIkNXFz5859L6XUrti+RgsyFRUVzJkzp7FOL0mSJKmJi4i3NrbPW8skSZIkZY5BRpIkSVLmGGQkSZIkZU6jrZGRJEmSilm3bh3V1dWsWbOmsUvRNtKqVSs6dOjATjvtVPIxBhlJkiQ1KdXV1ey6665UVFQQEY1djraylBLLli2jurqajh07lnyct5ZJkiSpSVmzZg177LGHIWYHERHsscceDZ6BM8hIkiSpyTHE7Fg25/tdb5CJiFsj4m8R8eJG9kdEXB8RiyLi+Yjo2eAqJEmSpIy56qqr+M1vftPYZTTY1KlTmTFjRoP7TZw4kSlTpmzN0hqklDUyk4BfABur+khgv/xXH+C/8n9KkiRJW2zCI6+WdbyxwzqVZZyHH36Yu+66qyxjbUtTp06ldevW9O/fv0H9Ro8evS3KK1m9MzIppSeB9zfR5RhgSsqZCXwpIvYqV4GSJEnStnTNNddw/fXXAzB27FgOPfRQAB577DFOOukkAFauXMnatWtp167d546dNWsW/fv3p0ePHvTv359XXnkFgEmTJjFmzJjafkcddRRTp04F4JZbbqFTp04MHjyYM844o7bfqaeeyllnncWQIUP48pe/zLRp0zjttNPo3Lkzp556au1YDz/8MP369aNnz56MGDGC1atXA1BRUcEll1xCz5496dq1Ky+//DKLFy9m4sSJTJgwge7duzN9+nTuu+8++vTpQ48ePRg6dCjvvPNO0X6XXnop48ePB6Cqqoq+ffvSrVs3jjvuOJYvXw7A4MGDueCCC+jduzedOnVi+vTp5fzWfE451si0B5YUbFfn2yRJkqTMOeSQQ2p/AJ8zZw6rV69m3bp1PPXUU1RWVgLw6KOPcthhh21w7AEHHMCTTz7J/Pnzufzyy7n44os3ea63336bK664gpkzZ/LII4/w8ssvf27/8uXLefzxx5kwYQLDhw9n7NixLFiwgBdeeIGqqiree+89rrzySh599FHmzZtHr169uPbaa2uPb9u2LfPmzeOss85i/PjxVFRUMHr0aMaOHUtVVRWVlZUMHDiQmTNnMn/+fEaOHMk111xTtF+hk08+mauvvprnn3+erl27ctlll9Xuq6mpYdasWVx33XWfay+3cjx+udjKnFS0Y8SZwJkA++yzTxlOXT7lnrLcGsa2uLuxSyjNkIsauwJJkqTNdtBBBzF37lxWrVpFy5Yt6dmzJ3PmzGH69Om1MzUPPvgg3/3udzc4dsWKFZxyyim89tprRATr1q3b5LlmzZrFoEGD2H333QEYMWIEr7762c+lw4cPJyLo2rUre+65J127dgXgwAMPZPHixVRXV/PSSy8xYMAAANauXUu/fv1qjz/++ONr39Mf//jHojVUV1dz4oknsnTpUtauXVvvI5BXrFjBBx98wKBBgwA45ZRTGDFiRNFzLl68eJNjbYlyzMhUA3sXbHcA3i7WMaV0c0qpV0qpV91pOEmSJKkp2GmnnaioqODXv/41/fv3p7KykieeeILXX3+dzp07A7kA0rt37w2O/fd//3eGDBnCiy++yH333Vf7SOEWLVrw6aef1vZb355S0d//12rZsiUAzZo1q329frumpoaUEsOGDaOqqoqqqipeeuklbrnllg2Ob968OTU1NUXPcfbZZzNmzBheeOEFbrrppi3+INJSzlkO5Qgy9wIn559e1hdYkVJaWoZxJUmSpEZxyCGHMH78eA455BAqKyuZOHEi3bt3JyJYsGABBxxwAM2bN9/guBUrVtC+fW6VxaRJk2rbKyoqqKqq4tNPP2XJkiXMmjULgN69ezNt2jSWL19OTU0Nd9/dsDtw+vbty9NPP82iRYsA+Oijjz43o1PMrrvuyqpVq4rWPHny5I32W69Nmzbstttutbff3XbbbbWzM9tSKY9f/h3wDLB/RFRHxOkRMToi1j+24AHgDWAR8Evge1utWkmSJGkbqKysZOnSpfTr148999yTVq1a1a4T+fOf/8wRRxxR9Lh//dd/5aKLLmLAgAF88sknte0DBgygY8eOdO3alfPOO4+ePXOfWNK+fXsuvvhi+vTpw9ChQ+nSpQtt2rQpuc527doxadIkRo0aRbdu3ejbt+8G62zqGj58OPfcc8/nFvGPGDGCyspK2rZtu9F+hSZPnsz5559Pt27dqKqqYty4cSXXXC5R33TW1tKrV680Z86cRjl3Ma6RKSPXyKipeuKqxq6gNF5Daqq8hrSNLFy4sPYWrqZo2LBhTJkyhb32auCDelcWv2lp9eoPad16F2pqajju26dx2kmjOG74kWWodDN9sXEeQFzs+x4Rc1NKvYr1L8dif0k7uCz8IgBgrP/HUxPlNSRtmXdWbtmajoa6/e77Nuu8e26k/dKrxvPotOmsWfMxhx86iGOPKj7bo8/zf0mSJElSIxr/40sau4RMKsdif0mSJEnapgwykiRJkjLHICNJkiQpcwwykiRJkjLHICNJkiRthut/9lPuvut3jV1G2UydPoMZz86uv9/UqcyYMaN2e+LEiUyZMmVrllaUTy2TJElS01bPZxjt8nFNg4b7sP/5W1JNralPPMrNk24vy1hNwdSnZtB6l13oP+zoTfebOpXWrVvTv39/AEaPHr0tytuAMzKSJElSgV9c9zN+NfEGAMZddD7fzH+uy/SpT/D9M74LwKqVK1m3di1t27ZjyV/e4oThRzKk/8GcMPxIqpf8ZYMx582dTf9hw+kxcBj9hw3nldcWATDpN3cy5ryLa/sd9a2TmTo9N9txy5Tf0qnnQAb/4zc54+zzavudetYPOGvshQw56gS+3K0v0556htO+P5bOBx/CqWf9oHashx+bSr+hw+lZeTgjTj6T1as/BKCia28u+Y+f0rPycLr2O5SXX32NxW8tYeKttzHhxl/SvXt3pk+fzn333UefPn3o0aMHQ4cO5Z133mHx4sVMnDiRCRMm1Pa79NJLGT9+PABVVVX07duXbt26cdxxx7F8+XIABg8ezAUXXEDv3r3p1KkT06dP3+Lvk0FGkiRJKtBvwEBmzngagOfmz+PDD1ezbt06np05gz79BgDw5NTHGThoCAAXn38uI0Z9mydmzOb4b43kRxf8cIMx99tvf5788z3Mf+oRLr/4fC6+7CebrOHtpX/lip9ex8xH7+ORP93By/ngs97yDz7g8ft+z4SrLmX4yFMY+70zWfDsVF5YsJCq51/kvWXLuHL8z3n0v+9k3vSH6dWjG9fecFPt8W332J150x/mrNNOZvz1E6nYd29Gn/Ydxn7vDKqqqqisrGTgwIHMnDmT+fPnM3LkSK655hoqKioYPXo0Y8eOre1X6OSTT+bqq6/m+eefp2vXrlx22WW1+2pqapg1axbXXXfd59o3l7eWSZIkSQW6de/J81XzWb1qFTvv3JKuX+vOc/Pn8uyMp7nymp8B8MSjjzDypO8AMHfWs9x6+x0AjBj5T1w57t82GHPlyhWMOOscXnv9TSKCdevWbbKGWXPnM2hAP3bffbfcuMcexauL3qjdP/zIw4kIunbpzJ7t2tH1wM4AHNh5fxb/ZQnVby/lpZdfZcDXc7eJrV27jn4HH1R7/PHDvwHAQT268cf7HihaQ3V1NSeeeCJLly5l7dq1dOzYcZM1r1ixgg8++IBBgwYBcMoppzBixIjPznn88blzHnQQixcv3uRYpXBGRpIkSSqw0047sfc++3LHb6bQq09f+vQbwNNPPsniN9+g0/4HADB/3hx6HHRw8QEiNmi6+seXM6SyPy/OfIL77pjMmo8/BqBFixZ8+mmq7bdmzRoAUtpgiM9pufPOADRr1oyWLXeubW/WrBk1NZ+QUmLYkEOoeupRqp56lJdmTeOWG6797Pj8Mc2bNafmk0+KnuPss89mzJgxvPDCC9x00021tW2uli1b5s7ZvDk1NQ1b11SMQUaSJEmqo++AgfzXf15Hv/4D6dt/AFN+/Uu+2q0bEcHLC1/iK/t1onnz5gD06tOXP939ewDuvusO+vTrt8F4q1auoP1eewEw6bd31rZX7NOBqhde5NNPP2VJ9f9j1rwqAHof1J1pTz/D8uUfUFNTw933Fp812Wj9Bx/E08/OZtHrbwLw0Ucf8eqi1zd5zK6tW7Nq9era7RUrVtC+fXsAJk+e/Fm/XXdl1apVGxzfpk0bdtttt9r1L7fddlvt7MzWYJCRJEmS6ujbbwDv/PWvHNS7D+3+bk9atmxVuz7m8UceYsjQw2v7Xnn1z7jj9ikM6X8wf7jjt1zxk/EbjPf9fzmXiy67igGHH80nn3xa2z6gb2867rsPXfsdynk/upyeX+sKQPt/2IuLzz2HPocdxdBjTqTL/p1o88Uvllx/u7Z7MOnG6xh1+vfo1v8w+g4dzsuvLtrkMcOPHMY99z/4uUX8I0aMoLKykrZt237Wb/hw7rnnntp+hSZPnsz5559Pt27dqKqqYty4cSXX3FCR6pu32kp69eqV5syZ0yjnLmbCI682dgn1Gtvi7sYuoTRDLmrsCrSNZeH6Aa8hNV1eQ2XmNZR5CxcupHPnziX3f2fllt3y1FDfOuYf+c+bfsWef79Xg47bk+UN6r969Ye0br0LNTU1HPft0zjtpFEcN/zIBo2xWb7YsPdVLsW+7xExN6XUq1h/F/tLkiRJDXDXf//fbXKeS68az6PTprNmzcccfuggjs0/Blo5BhlJkiSpCRr/40sau4QmzTUykiRJkjLHICNJkqQmp7HWcatxbM732yAjSZKkJqVVq1YsW7bMMLODSCmxbNkyWrVq1aDjXCMjSZKkJqVDhw5UV1fz7rvvltR/5f+s28oVlcf78VFjl1CaVh9s+1O2akWHDh0adIxBRpIkSU3KTjvtRMeOHUvun51HmG+bp51tsYw8wtxbyyRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRlTklBJiKOiIhXImJRRFxYZP8+EfFERMyPiOcj4hvlL1WSJEmScuoNMhHRHLgBOBLoAoyKiC51uv0IuCul1AMYCdxY7kIlSZIkab1SZmR6A4tSSm+klNYCdwDH1OmTgC/mX7cB3i5fiZIkSZL0eS1K6NMeWFKwXQ30qdPnUuDhiDgb2AUYWpbqJEmSJKmIUmZkokhbqrM9CpiUUuoAfAO4LSI2GDsizoyIOREx59133214tZIkSZJEaUGmGti7YLsDG946djpwF0BK6RmgFdC27kAppZtTSr1SSr3atWu3eRVLkiRJ2uGVEmRmA/tFRMeI2JncYv576/T5C3AYQER0JhdknHKRJEmStFXUG2RSSjXAGOAhYCG5p5MtiIjLI+LofLcfAmdExHPA74BTU0p1bz+TJEmSpLIoZbE/KaUHgAfqtI0reP0SMKC8pUmSJElScSV9IKYkSZIkNSUGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDklBZmIOCIiXomIRRFx4Ub6fCsiXoqIBRHx2/KWKUmSJEmfaVFfh4hoDtwADAOqgdkRcW9K6aWCPvsBFwEDUkrLI+LvtlbBkiRJklTKjExvYFFK6Y2U0lrgDuCYOn3OAG5IKS0HSCn9rbxlSpIkSdJnSgky7YElBdvV+bZCnYBOEfF0RMyMiCOKDRQRZ0bEnIiY8+67725exZIkSZJ2eKUEmSjSlupstwD2AwYDo4BfRcSXNjgopZtTSr1SSr3atWvX0FolSZIkCSgtyFQDexdsdwDeLtLnv1NK61JKbwKvkAs2kiRJklR2pQSZ2cB+EdExInYGRgL31unzJ2AIQES0JXer2RvlLFSSJEmS1qs3yKSUaoAxwEPAQuCulNKCiLg8Io7Od3sIWBYRLwFPAOenlJZtraIlSZIk7djqffwyQErpAeCBOm3jCl4n4Nz8lyRJkiRtVSV9IKYkSZIkNSUGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZY5CRJEmSlDkGGUmSJEmZU1KQiYgjIuKViFgUERduot8JEZEiolf5SpQkSZKkz6s3yEREc+AG4EigCzAqIroU6bcrcA7wbLmLlCRJkqRCpczI9AYWpZTeSCmtBe4AjinS7wrgGmBNGeuTJEmSpA2UEmTaA0sKtqvzbbUiogewd0rp/jLWJkmSJElFlRJkokhbqt0Z0QyYAPyw3oEizoyIOREx59133y29SkmSJEkqUEqQqQb2LtjuALxdsL0r8FVgakQsBvoC9xZb8J9Sujml1Cul1Ktdu3abX7UkSZKkHVopQWY2sF9EdIyInYGRwL3rd6aUVqSU2qaUKlJKFcBM4OiU0pytUrEkSZKkHV69QSalVAOMAR4CFgJ3pZQWRMTlEXH01i5QkiRJkupqUUqnlNIDwAN12sZtpO/gLS9LkiRJkjaupA/ElCRJkqSmxCAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyxyAjSZIkKXMMMpIkSZIyp6QgExFHRMQrEbEoIi4ssv/ciHgpIp6PiMciYt/ylypJkiRJOfUGmYhoDtwAHAl0AUZFRJc63eYDvVJK3YA/ANeUu1BJkiRJWq+UGZnewKKU0hsppbXAHcAxhR1SSk+klD7Kb84EOpS3TEmSJEn6TClBpj2wpGC7Ot+2MacDf96SoiRJkiRpU1qU0CeKtKWiHSNOAnoBgzay/0zgTIB99tmnxBIlSZIk6fNKmZGpBvYu2O4AvF23U0QMBf4NODql9HGxgVJKN6eUeqWUerVr125z6pUkSZKkkoLMbGC/iOgYETsDI4F7CztERA/gJnIh5m/lL1OSJEmSPlNvkEkp1QBjgIeAhcBdKaUFEXF5RByd7/ZToDXw+4ioioh7NzKcJEmSJG2xUtbIkFJ6AHigTtu4gtdDy1yXJEmSJG1USR+IKUmSJElNiUFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRljkFGkiRJUuYYZCRJkiRlTklBJiKOiIhXImJRRFxYZH/LiLgzv//ZiKgod6GSJEmStF69QSYimgM3AEcCXYBREdGlTrfTgeUppa8AE4Cry12oJEmSJK1XyoxMb2BRSumNlNJa4A7gmDp9jgEm51//ATgsIqJ8ZUqSJEnSZ0oJMu2BJQXb1fm2on1SSjXACmCPchQoSZIkSXVFSmnTHSJGAF9PKf1zfvs7QO+U0tkFfRbk+1Tnt1/P91lWZ6wzgTPzm/sDr5Trjewg2gLvNXYRUoZ5DUlbxmtI2jJeQw23b0qpXbEdLUo4uBrYu2C7A/D2RvpUR0QLoA3wft2BUko3AzeXUrE2FBFzUkq9GrsOKau8hqQt4zUkbRmvofIq5day2cB+EdExInYGRgL31ulzL3BK/vUJwOOpvqkeSZIkSdpM9c7IpJRqImIM8BDQHLg1pbQgIi4H5qSU7gVuAW6LiEXkZmJGbs2iJUmSJO3YSrm1jJTSA8ADddrGFbxeA4wob2kqwtvypC3jNSRtGa8hact4DZVRvYv9JUmSJKmpKWWNjCRJkiQ1KQYZSTuMiJhRz/4HIuJL26oeaXsTERUR8WL+9eCIuL+xa5IaU0ScExELI+LuiHgmIj6OiPMau67tRUlrZCSpqYmI5imlTxpyTEqpfz37v7FlVUnZFBFB7nbzTxu7Fmk78z3gSOBDYF/g2MYtZ/vijOCji/EAAAP5SURBVEwTExF/ioi5EbEg/wGiRMTqgv0nRMSk/Os9I+KeiHgu/7XJH9KkrMj/VvfliJgcEc9HxB8i4gsRsTgixkXEU8CIiPjfEfFg/pqZHhEH5I8vem2sv5YiYq+IeDIiqiLixYiozLcvjoi2+dfn5ve9GBE/KKhrYUT8Mn+NPhwR/6tR/iNJW6jg7/ONwDzgO/nfGM+LiN9HROt8v4MjYkb+WpoVEbvmj52e7zvPf3+kDUXERODL5D6m5NsppdnAusatavvijEzTc1pK6f38D0ezI+LuTfS9HpiWUjouIpoDrbdNidI2sT9wekrp6Yi4ldxvtQDWpJQGAkTEY8DolNJrEdEHuBE4lPqvjX8CHkop/Ti//wuFOyPiIOC7QB8ggGcjYhqwHNgPGJVSOiMi7gK+Cdxe9ncvbRv7k/u7Pg74IzA0pfRhRFwAnBsRPwHuBE5MKc2OiC8C/wP8DRiWUloTEfsBvwP8kD+pQEppdEQcAQxJKb3X2PVsjwwyTc85EXFc/vXe5H5o2phDgZMB8rfYrNjKtUnb0pKU0tP517cD5+Rf3wmQ/21xf+D3ubtiAGiZ/7O+a2M2cGtE7AT8KaVUVWf/QOCelNKH+XP9Eagk91u1Nwv6zwUqtuA9So3trZTSzIg4CugCPJ2/nnYGniEXdJbmf5NMSmklQETsAvwiIroDnwCdGqN4STs2g0wTEhGDgaFAv5TSRxExFWgFFD4ju1UjlCY1hrrPhl+//WH+z2bAByml7g0eOKUnI+IQ4B/JfZjvT1NKUwq6xEYOBfi44PUngLeWKcvWX08BPJJSGlW4MyK6seG1CDAWeAf4Grlrcc3WLFKSinGNTNPSBlieDzEHAH3z7e9EROeIaAYcV9D/MeAsyC18zk/5S9uLfSKiX/71KOCpwp353wy/GREjILdYOSK+lt+9yWsjIvYF/pZS+iVwC9CzzrmfBI7Nr8vZhdx1N718b01qcmYCAyLiKwD5v/udgJeBf4iIg/Ptu0ZEC3L/Xi3NPxzgO0DzRqpb0g7MINO0PAi0iIjngSvI/cMCcCFwP/A4sLSg/78AQyLiBXK3uBy4DWuVtraFwCn562F34L+K9Pk2cHpEPAcsAI7Jt9d3bQwGqiJiPrk1Lj8v3JlSmgdMAmYBzwK/SinNL8N7kpqklNK7wKnA7/LX3EzggJTSWuBE4D/z19kj5O4MuJHc9TmT3G1lHxYdWBIAEfH3EVENnAv8KCKq/QX0louUis0YS1LjiYgK4P6U0lcbuRRJktREOSMjSZIkKXOckZEkSZKUOc7ISJIkScocg4wkSZKkzDHISJIkScocg4wkSZKkzDHISJIkScocg4wkSZKkzPn/SYNToTLtSLoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PERFORMANCE ON THE FAKE TEST ###\n",
    "\n",
    "plt.figure(figsize=(14,5))\n",
    "plt.bar(np.arange(len(nn_train_aug_new))-0.1, nn_train_aug_new.values(), \n",
    "        alpha=0.5, width=0.2, label='w/ augmentation')\n",
    "plt.bar(np.arange(len(nn_train_new))+0.1, nn_train_new.values(), \n",
    "        alpha=0.5, width=0.2, label='w/o augmentation')\n",
    "plt.xticks(range(len(nn_train_aug_new)), nn_train_aug_new.keys())\n",
    "plt.title('Neural Network'); plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAE/CAYAAACdNPbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dfZhV5Xnv8e8toKRCSCLUKmgGGwmiQxUJrw6CijWpJJITqhyNoIle2KCn2ESjJzH4klopRxLTeNAWRY2NsdEkmvoCqCjGEORlogISUUmc6jFKEUVDYPQ+f+zNZBgG2MjAzJLv57q4XGs9z3rWvfewZP/mWWuvyEwkSZIkqUj2au0CJEmSJGlHGWQkSZIkFY5BRpIkSVLhGGQkSZIkFY5BRpIkSVLhGGQkSZIkFY5BRpL2UBFxekTMep/7Lo2I4S1cUpsXEfdHxLjWrkOSBOFzZCSp7YuIVcCXM3NOKxx7JlCXmd/YyXGqgBeBt8ubXgemZ+Y/7cy4kqQ9U/vWLkCStMf5SGbWR0R/4NGIWJSZs1vyABHRPjPrW3JMSVLb4qVlklRwEXFORKyMiP+OiHsi4sBGbSdGxIqIWBsR10fEoxHx5XLb+Ih4vLwcETEtIn5f7vtURBwREecCpwMXRcS6iLi33H9VRJxQXm4XEZdGxPMR8VZELIqIg7ZXd2YuBJYCRzaq98CIuCsiXouIFyPigkZtH4qIWyJiTUQsj4iLIqKuUfuqiLg4Ip4C3o6I9tsZb0BELIyINyPi1Yi4try9Y0T8ICJWR8QbEfFkROxfbpvb6P3bKyK+ERG/Lb9vt0ZEl3JbVURkRIyLiN9FxOsR8b93+IcrSdoqg4wkFVhEHAdcDfwtcADwW+COcltX4MfAJcB+wApgyFaGOhEYBvQCPgKcCqzOzBuB24EpmdkpM0c1s++FwFjgM8CHgbOBdyqofRBwBLCyvL4XcC/wa6A7cDzw9xHx1+VdvgVUAYcAI4Ezmhl2LPA35dfw3nbG+y7w3cz8MPCXwJ3l7eOALsBBlN63CcAfmjnW+PKfEeWaOgH/0qTPMcAny8e+LCIO29Z7IkmqnEFGkortdOCmzFycmX+kFFoGl+9H+QywNDPvLl9mdR3w/7YyzkagM9Cb0v2TyzPzlQpr+DLwjcxckSW/zszV2+j/ekT8AfglcD3w0/L2TwHdMvOKzNyQmS8A/wqcVm7/W+AfM3NNZtaVX09T12XmS5n5hwrG2wh8IiK6Zua6zJzfaPt+wCcy893MXJSZbzZzrNOBazPzhcxcR+m9Py0iGl+2fXlm/iEzf00pUP3VNt4XSdIOMMhIUrEdSGkWBoDyB+rVlGYgDgReatSWQF3TAcptD1OaTfg+8GpE3BgRH66whoOA53eg5q6UZi++CgwHOpS3fxw4sHw51xsR8QZwKbB/uX2z19Nkublt2xvvS5RmoJ4tXz52cnn7bcCDwB0R8XJETImIDmxps/e+vNy+0fiweXB8p/y6JUktwCAjScX2MqUP7ABExL6UZhP+C3gF6NGoLRqvN5WZ12Xm0cDhlD7gf21T03ZqeInSpVkVK890/B9gPfB3jcZ5MTM/0uhP58z8TLl9s9dDKUBtMXSTurY6XmY+l5ljgT8HrgF+HBH7ZubGzLw8M/tQuhTvZODMZo612XsPHAzUA6/uwFshSXqfDDKSVBwdyjeib/rTHvh34KyIODIi9gH+EfhVZq4C/hOojohTyn2/AvxFcwNHxKciYmB55uFtSgHj3XLzq5TuAdmafwOujIhDy18a0Dci9qvwNf0TpS8S6AgsAN4s37D/ofKXCBwREZ8q970TuCQiPhoR3YGJ2xl7m+NFxBkR0S0z3wPeKO/zbkSMiIjqiGgHvEnpUrN3mxn/h8CkiOgZEZ0ovfc/8tvSJGn3MMhIUnHcR+mm801/JmfmQ8A3gbsozVj8JeV7QDLzdWAMMIXS5WZ9gIXAH5sZ+8OU7h9ZQ+kSqdXA1HLbDKBP+fKsnzaz77WUQsYsSh/8ZwAfqvA1/Wf5mOdk5rvAKErfYvYipefM/BulG+8BrqB0adyLwBxKX2TQ3GsBSrM+2xnvJGBpRKyjdOP/aZm5nlLY+3H5tSwHHgV+0MwhbqJ0Gdpj5fHXA+dX+LolSTvJB2JK0h6i/K1gdcDpmflIa9ezsyLiPErh49jWrkWStPs5IyNJH2AR8dcR8ZHyZWeXAgHM385ubVJEHBARQ8vPb/kk8A/AT1q7LklS62i//S6SpAIbTOk+mr2BZcAp5a8mLqK9gRuAnpTuabmD0tc3S5L2QF5aJkmSJKlwvLRMkiRJUuEYZCRJkiQVTqvdI9O1a9esqqpqrcNLkiRJauMWLVr0emZ2a66t1YJMVVUVCxcubK3DS5IkSWrjIuK3W2vz0jJJkiRJhWOQkSRJklQ4BhlJkiRJheMDMSVJktSmbNy4kbq6OtavX9/apWg36dixIz169KBDhw4V72OQkSRJUptSV1dH586dqaqqIiJauxztYpnJ6tWrqauro2fPnhXv56VlkiRJalPWr1/PfvvtZ4jZQ0QE++233w7PwBlkJEmS1OYYYvYs7+fnbZCRJEmS3oerr76a22+/vbXL2GFz587liSee2OF+06dP59Zbb92Vpe0Q75GRJElSmzZt9m9adLxJI3u1yDizZs3izjvvbJGxdqe5c+fSqVMnhgwZskP9JkyYsDvKq5gzMpIkSVIjU6ZM4brrrgNg0qRJHHfccQA89NBDnHHGGQC8+eabbNiwgW7dum2274IFCxgyZAhHHXUUQ4YMYcWKFQDMnDmTiRMnNvQ7+eSTmTt3LgAzZsygV69eDB8+nHPOOaeh3/jx4znvvPMYMWIEhxxyCI8++ihnn302hx12GOPHj28Ya9asWQwePJh+/foxZswY1q1bB0BVVRXf+ta36NevH9XV1Tz77LOsWrWK6dOnM23aNI488kjmzZvHvffey8CBAznqqKM44YQTePXVV5vtN3nyZKZOnQpAbW0tgwYNom/fvowePZo1a9YAMHz4cC6++GIGDBhAr169mDdvXkv+aDZjkJEkSZIaGTZsWMMH8IULF7Ju3To2btzI448/Tk1NDQBz5szh+OOP32Lf3r1789hjj7FkyRKuuOIKLr300m0e6+WXX+bKK69k/vz5zJ49m2effXaz9jVr1vDwww8zbdo0Ro0axaRJk1i6dClPP/00tbW1vP7661x11VXMmTOHxYsX079/f6699tqG/bt27crixYs577zzmDp1KlVVVUyYMIFJkyZRW1tLTU0NxxxzDPPnz2fJkiWcdtppTJkypdl+jZ155plcc801PPXUU1RXV3P55Zc3tNXX17NgwQK+853vbLa9pXlpWVlLT1nuCi01DSpJkqStO/roo1m0aBFvvfUW++yzD/369WPhwoXMmzevYabmgQce4Kyzztpi37Vr1zJu3Diee+45IoKNGzdu81gLFizg2GOP5WMf+xgAY8aM4Te/+dPn0lGjRhERVFdXs//++1NdXQ3A4YcfzqpVq6irq2PZsmUMHToUgA0bNjB48OCG/T//+c83vKa777672Rrq6uo49dRTeeWVV9iwYcN2vwJ57dq1vPHGGxx77LEAjBs3jjFjxjR7zFWrVm1zrJ3hjIwkSZLUSIcOHaiqquLmm29myJAh1NTU8Mgjj/D8889z2GGHAaUAMmDAgC32/eY3v8mIESN45plnuPfeexu+Urh9+/a89957Df02bc/Mbdayzz77ALDXXns1LG9ar6+vJzMZOXIktbW11NbWsmzZMmbMmLHF/u3ataO+vr7ZY5x//vlMnDiRp59+mhtuuGGnH0RayTFbgjMyRfLI1a1dQWVGXNLaFUiSJO2UYcOGMXXqVG666Saqq6u58MILOfroo4kIli5dSu/evWnXrt0W+61du5bu3bsDpftiNqmqquL666/nvffe47/+679YsGABAAMGDGDSpEmsWbOGzp07c9dddzXMulRi0KBBfOUrX2HlypV84hOf4J133qGuro5evbZ+JU/nzp158803m635lltu2Wq/Tbp06cJHP/pR5s2bR01NDbfddlvD7MzuZJCRtOfwlwGSpArV1NTw7W9/m8GDB7PvvvvSsWPHhvtE7r//fk466aRm97vooosYN24c1157bcOXBAAMHTqUnj17Ul1dzRFHHEG/fv0A6N69O5deeikDBw7kwAMPpE+fPnTp0qXiOrt168bMmTMZO3Ysf/zjHwG46qqrthlkRo0axRe+8AV+9rOf8b3vfY/JkyczZswYunfvzqBBg3jxxReb7dfYLbfcwoQJE3jnnXc45JBDuPnmmyuuuaXE9qazdpX+/fvnwoULW+XYzSnEPTLt72rtEirjhzC1VQYZSSqE5cuXN1zC1RaNHDmSW2+9lQMOOKBFxlu3bh2dOnWivr6e0aNHc/bZZzN69OgWGbtImvu5R8SizOzfXH9nZCRJkqQdMHv27BYdb/LkycyZM4f169dz4okncsopp7To+B9UBhlJkiSpFW16Not2jEFG0k4rwqWZAJP8P54kSR8Yfv2yJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEnvw9VXX83tt9/e2mW0mLlz5/LEE0/scL/p06dz66237srSmuWtr5IkSWrbWvo5YC30vK5Zs2Zx5513tshYbcHcuXPp1KkTQ4YM2aF+EyZM2B3lbcEZGUmSJKmRKVOmcN111wEwadIkjjvuOAAeeughzjjjDADefPNNNmzYQLdu3fjtb3/L8ccfT9++fTn++OP53e9+t8WYCxYsYMiQIRx11FEMGTKEFStWADBz5kwmTpzY0O/kk09m7ty5AMyYMYNevXoxfPhwzjnnnIZ+48eP57zzzmPEiBEccsghPProo5x99tkcdthhjB8/vmGsWbNmMXjwYPr168eYMWNYt24dAFVVVXzrW9+iX79+VFdX8+yzz7Jq1SqmT5/OtGnTOPLII5k3bx733nsvAwcO5KijjuKEE07g1Vdfbbbf5MmTG75Cura2lkGDBtG3b19Gjx7NmjVrABg+fDgXX3wxAwYMoFevXsybN2+nf04GGUmSJKmRYcOGNXzQXrhwIevWrWPjxo08/vjj1NTUADBnzhyOP/54ACZOnMiZZ57JU089xemnn84FF1ywxZi9e/fmscceY8mSJVxxxRVceuml26zh5Zdf5sorr2T+/PnMnj2bZ599drP2NWvW8PDDDzNt2jRGjRrFpEmTWLp0KU8//TS1tbW8/vrrXHXVVcyZM4fFixfTv39/rr322ob9u3btyuLFiznvvPOYOnUqVVVVTJgwgUmTJlFbW0tNTQ3HHHMM8+fPZ8mSJZx22mlMmTKl2X6NnXnmmVxzzTU89dRTVFdXc/nllze01dfXs2DBAr7zne9stv398tIySZIkqZGjjz6aRYsW8dZbb7HPPvvQr18/Fi5cyLx58xpmah544AHOOussAH75y19y9913A/DFL36Riy66aIsx165dy7hx43juueeICDZu3LjNGhYsWMCxxx7Lxz72MQDGjBnDb37zp+e2jRo1ioigurqa/fffn+rqagAOP/xwVq1aRV1dHcuWLWPo0KEAbNiwgcGDBzfs//nPf77htW6qvam6ujpOPfVUXnnlFTZs2EDPnj23WfPatWt54403OPbYYwEYN24cY8aMafaYq1at2uZYlXBGRpIkSWqkQ4cOVFVVcfPNNzNkyBBqamp45JFHeP755znssMOAUtAYMGBAs/tHxBbbvvnNbzJixAieeeYZ7r33XtavXw9A+/btee+99xr6bdqemduscZ999gFgr732aljetF5fX09mMnLkSGpra6mtrWXZsmXMmDFji/3btWtHfX19s8c4//zzmThxIk8//TQ33HBDQ23vVyXH3BEGGUmSJKmJYcOGMXXqVIYNG0ZNTQ3Tp0/nyCOPJCJYunQpvXv3pl27dgAMGTKEO+64A4Dbb7+dY445Zovx1q5dS/fu3YHSfTGbVFVVUVtby3vvvcdLL73EggULABgwYACPPvooa9asob6+nrvuumuH6h80aBC/+MUvWLlyJQDvvPPOZjM6zencuTNvvfVWszXfcsstW+23SZcuXfjoRz/acFnebbfd1jA7sysYZCRJkqQmampqeOWVVxg8eDD7778/HTt2bLgf5P777+ekk05q6Hvddddx880307dvX2677Ta++93vbjHeRRddxCWXXMLQoUN59913G7YPHTqUnj17Ul1dzVe/+lX69esHQPfu3bn00ksZOHAgJ5xwAn369KFLly4V19+tWzdmzpzJ2LFj6du3L4MGDdriPpumRo0axU9+8pPNbuIfM2YMNTU1dO3adav9Grvlllv42te+Rt++famtreWyyy6ruOYdFdubttpV+vfvnwsXLmyVYzdn2uxtJ9S2YFL7HUviraaFvtJQxVGE8wc8hySpKJYvX95wCVdbNHLkSG699VYOOOCAXXqcdevW0alTJ+rr6xk9ejRnn302o0eP3qXHbE3N/dwjYlFm9m+uvzf7S5IkSTtg9uzZu+U4kydPZs6cOaxfv54TTzyRU045ZbcctygMMpIkSVIbtOnZLGqe98hIkiRJKhyDjCRJktqc1rqPW63j/fy8DTKSJElqUzp27Mjq1asNM3uIzGT16tV07Nhxh/bzHhlJkiS1KT169KCuro7XXnuttUvRbtKxY0d69OixQ/sYZCRJktSmdOjQgZ49e7Z2GWrjvLRMkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuFUFGQi4qSIWBERKyPi6820HxwRj0TEkoh4KiI+0/KlSpIkSVLJdoNMRLQDvg98GugDjI2IPk26fQO4MzOPAk4Drm/pQiVJkiRpk0pmZAYAKzPzhczcANwBfK5JnwQ+XF7uArzcciVKkiRJ0uYqeSBmd+ClRut1wMAmfSYDsyLifGBf4IQWqU6SJEmSmlHJjEw0sy2brI8FZmZmD+AzwG0RscXYEXFuRCyMiIWvvfbajlcrSZIkSVQWZOqAgxqt92DLS8e+BNwJkJm/BDoCXZsOlJk3Zmb/zOzfrVu391exJEmSpD1eJUHmSeDQiOgZEXtTupn/niZ9fgccDxARh1EKMk65SJIkSdolthtkMrMemAg8CCyn9O1kSyPiioj4bLnbPwDnRMSvgR8C4zOz6eVnkiRJktQiKrnZn8y8D7ivybbLGi0vA4a2bGmSJEmS1LyKHogpSZIkSW1JRTMykiRJUls1bfZvWruEikwa2au1S/hAcUZGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuH4HBlJkiRpd3jk6tauoDIjLmntCirijIwkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwvFbyyRJamXTZv+mtUuoyKSRvVq7BElq4IyMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHB+IKUmSKvPI1a1dQWVGXNLaFUjaDZyRkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhVNRkImIkyJiRUSsjIivb6XP30bEsohYGhH/3rJlSpIkSdKftN9eh4hoB3wfGAnUAU9GxD2ZuaxRn0OBS4ChmbkmIv58VxUsSZIkSZXMyAwAVmbmC5m5AbgD+FyTPucA38/MNQCZ+fuWLVOSJEmS/qSSINMdeKnRel15W2O9gF4R8YuImB8RJ7VUgZIkSZLU1HYvLQOimW3ZzDiHAsOBHsC8iDgiM9/YbKCIc4FzAQ4++OAdLlaSJEmSoLIZmTrgoEbrPYCXm+nzs8zcmJkvAisoBZvNZOaNmdk/M/t369bt/dYsSZIkaQ9XSZB5Ejg0InpGxN7AacA9Tfr8FBgBEBFdKV1q9kJLFipJkiRJm2w3yGRmPTAReBBYDtyZmUsj4oqI+Gy524PA6ohYBjwCfC0zV++qoiVJkiTt2Sq5R4bMvA+4r8m2yxotJ3Bh+Y8kSZIk7VIVPRBTkiRJktoSg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSqcioJMRJwUESsiYmVEfH0b/b4QERkR/VuuREmSJEna3HaDTES0A74PfBroA4yNiD7N9OsMXAD8qqWLlCRJkqTGKpmRGQCszMwXMnMDcAfwuWb6XQlMAda3YH2SJEmStIVKgkx34KVG63XlbQ0i4ijgoMz8+bYGiohzI2JhRCx87bXXdrhYSZIkSYLKgkw0sy0bGiP2AqYB/7C9gTLzxszsn5n9u3XrVnmVkiRJktRIJUGmDjio0XoP4OVG652BI4C5EbEKGATc4w3/kiRJknaVSoLMk8ChEdEzIvYGTgPu2dSYmWszs2tmVmVmFTAf+GxmLtwlFUuSJEna4203yGRmPTAReBBYDtyZmUsj4oqI+OyuLlCSJEmSmmpfSafMvA+4r8m2y7bSd/jOlyVJkiRJW1fRAzElSZIkqS0xyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqnIqCTEScFBErImJlRHy9mfYLI2JZRDwVEQ9FxMdbvlRJkiRJKtlukImIdsD3gU8DfYCxEdGnSbclQP/M7Av8GJjS0oVKkiRJ0iaVzMgMAFZm5guZuQG4A/hc4w6Z+UhmvlNenQ/0aNkyJUmSJOlPKgky3YGXGq3XlbdtzZeA+3emKEmSJEnalvYV9IlmtmWzHSPOAPoDx26l/VzgXICDDz64whIlSZIkaXOVzMjUAQc1Wu8BvNy0U0ScAPxv4LOZ+cfmBsrMGzOzf2b279at2/upV5IkSZIqCjJPAodGRM+I2Bs4DbincYeIOAq4gVKI+X3LlylJkiRJf7LdIJOZ9cBE4EFgOXBnZi6NiCsi4rPlbv8MdAL+IyJqI+KerQwnSZIkSTutkntkyMz7gPuabLus0fIJLVyXJEmSJG1VRQ/ElCRJkqS2xCAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKp6IgExEnRcSKiFgZEV9vpn2fiPhRuf1XEVHV0oVKkiRJ0ibbDTIR0Q74PvBpoA8wNiL6NOn2JWBNZn4CmAZc09KFSpIkSdImlczIDABWZuYLmbkBuAP4XJM+nwNuKS//GDg+IqLlypQkSZKkP6kkyHQHXmq0Xlfe1myfzKwH1gL7tUSBkiRJktRUZOa2O0SMAf46M79cXv8iMCAzz2/UZ2m5T115/flyn9VNxjoXOLe8+klgRUu9kD1EV+D11i5CKjDPIWnneA5JO8dzaMd9PDO7NdfQvoKd64CDGq33AF7eSp+6iGgPdAH+u+lAmXkjcGMlFWtLEbEwM/u3dh1SUXkOSTvHc0jaOZ5DLauSS8ueBA6NiJ4RsTdwGnBPkz73AOPKy18AHs7tTfVIkiRJ0vu03RmZzKyPiInAg0A74KbMXBoRVwALM/MeYAZwW0SspDQTc9quLFqSJEnSnq2SS8vIzPuA+5psu6zR8npgTMuWpmZ4WZ60czyHpJ3jOSTtHM+hFrTdm/0lSZIkqa2p5B4ZSZIkSWpTDDKS9hgR8cR22u+LiI/srnqkD5qIqIqIZ8rLwyPi561dk9SaIuKCiFgeEXdFxC8j4o8R8dXWruuDopJ4s1gAAAQsSURBVKJ7ZCSprYmIdpn57o7sk5lDttP+mZ2rSiqmiAhKl5u/19q1SB8wfwd8Gngb+DhwSuuW88HijEwbExE/jYhFEbG0/ABRImJdo/YvRMTM8vL+EfGTiPh1+c82P6RJRVH+re6zEXFLRDwVET+OiD+LiFURcVlEPA6MiYi/jIgHyufMvIjoXd6/2XNj07kUEQdExGMRURsRz0RETXn7qojoWl6+sNz2TET8faO6lkfEv5bP0VkR8aFWeZOkndTo7/P1wGLgi+XfGC+OiP+IiE7lfp+KiCfK59KCiOhc3ndeue9i//2RthQR04FDKD2m5PTMfBLY2LpVfbA4I9P2nJ2Z/13+cPRkRNy1jb7XAY9m5uiIaAd02j0lSrvFJ4EvZeYvIuImSr/VAlifmccARMRDwITMfC4iBgLXA8ex/XPjfwIPZua3y+1/1rgxIo4GzgIGAgH8KiIeBdYAhwJjM/OciLgT+B/AD1r81Uu7xycp/V2/DLgbOCEz346Ii4ELI+KfgB8Bp2bmkxHxYeAPwO+BkZm5PiIOBX4I+JA/qZHMnBARJwEjMvP11q7ng8gg0/ZcEBGjy8sHUfrQtDXHAWcClC+xWbuLa5N2p5cy8xfl5R8AF5SXfwRQ/m3xEOA/SlfFALBP+b/bOzeeBG6KiA7ATzOztkn7McBPMvPt8rHuBmoo/VbtxUb9FwFVO/Eapdb228ycHxEnA32AX5TPp72BX1IKOq+Uf5NMZr4JEBH7Av8SEUcC7wK9WqN4SXs2g0wbEhHDgROAwZn5TkTMBToCjb8ju2MrlCa1hqbfDb9p/e3yf/cC3sjMI3d44MzHImIY8DeUHub7z5l5a6MusZVdAf7YaPldwEvLVGSbzqcAZmfm2MaNEdGXLc9FgEnAq8BfUToX1+/KIiWpOd4j07Z0AdaUQ0xvYFB5+6sRcVhE7AWMbtT/IeA8KN34XJ7ylz4oDo6IweXlscDjjRvLvxl+MSLGQOlm5Yj4q3LzNs+NiPg48PvM/FdgBtCvybEfA04p35ezL6Xzbl7LvTSpzZkPDI2ITwCU/+73Ap4FDoyIT5W3d46I9pT+vXql/OUAXwTatVLdkvZgBpm25QGgfUQ8BVxJ6R8WgK8DPwceBl5p1P9/ASMi4mlKl7gcvhtrlXa15cC48vnwMeD/NtPndOBLEfFrYCnwufL27Z0bw4HaiFhC6R6X7zZuzMzFwExgAfAr4N8yc0kLvCapTcrM14DxwA/L59x8oHdmbgBOBb5XPs9mU7oy4HpK5+d8SpeVvd3swJIAiIi/iIg64ELgGxFR5y+gd15kNjdjLEmtJyKqgJ9n5hGtXIokSWqjnJGRJEmSVDjOyEiSJEkqHGdkJEmSJBWOQUaSJElS4RhkJEmSJBWOQUaSJElS4RhkJEmSJBWOQUaSJElS4fx/H8PVOUbzNpwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PERFORMANCE ON THE STANDARD TEST ###\n",
    "\n",
    "plt.figure(figsize=(14,5))\n",
    "plt.bar(np.arange(len(logistic_train_aug))-0.1, logistic_train_aug.values(), \n",
    "        alpha=0.5, width=0.2, label='w/ augmentation')\n",
    "plt.bar(np.arange(len(logistic_train))+0.1, logistic_train.values(), \n",
    "        alpha=0.5, width=0.2, label='w/o augmentation')\n",
    "plt.xticks(range(len(logistic_train_aug)), logistic_train_aug.keys())\n",
    "plt.title('Logistic Regression'); plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAE/CAYAAACdNPbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dfZxWdZ3/8ddHQKkwKmFdA2uwRMWGBJFbUSh0tZWUNhLWUrL0hxu2P0wz2SLvdkvXxHXXFi0KtTaz1FZbSyUFSUPkZrzhLkkp5idrSAiiEYx+fn9cF+MwDMyFDMwceT0fj3l4nXO+53s+1zWch9d7vud7TmQmkiRJklQk+7R2AZIkSZK0swwykiRJkgrHICNJkiSpcAwykiRJkgrHICNJkiSpcAwykiRJkgrHICNJe6mIOCMi7n+T+y6KiGEtXFKbFxG/iIizWrsOSRKEz5GRpLYvIlYAn8/MGa1w7OlAbWZ+dRf7qQKeA14pr3oRmJqZ39yVfiVJe6f2rV2AJGmv867MrIuIfsCsiJifmQ+05AEion1m1rVkn5KktsVLyySp4CLinIhYHhF/ioi7I+K9DbadGBHLImJdRHw7ImZFxOfL28ZFxK/LryMipkTEH8ttn4yID0XEucAZwJcjYkNE3FNuvyIiRpRft4uISRHxu4h4OSLmR8TBzdWdmfOARcBRDep9b0TcERGrI+K5iPhig21vi4ibI2JtRCyJiC9HRG2D7Ssi4uKIeBJ4JSLaN9Nf/4iYFxHrI+KFiLi2vL5jRPwgItZExEsR8XhEHFjeNrPB57dPRHw1In5f/txuiYjO5W1VEZERcVZE/CEiXoyIf9rpX64kabsMMpJUYBHxEeAbwKeAg4DfA7eVt3UBfgpcAhwALAMGb6erE4HjgJ7Au4DTgTWZeRPwQ+DqzOyUmSOb2PcCYCzwMeCdwNnAqxXUPhD4ELC8vLwPcA/wBNAN+CjwfyPib8q7fB2oAg4BTgA+3US3Y4G/Lb+H15vp79+Af8vMdwIfAG4vrz8L6AwcTOlzGw/8uYljjSv/DC/X1An4j0ZtjgUOKx97ckQcsaPPRJJUOYOMJBXbGcD3MnNBZv6FUmgZVJ6P8jFgUWbeWb7M6nrgf7fTz2Zgf+BwSvMnl2Tmqgpr+Dzw1cxcliVPZOaaHbR/MSL+DPwG+Dbws/L6Y4CumXl5Zm7KzGeB7wBjyts/BfxLZq7NzNry+2ns+sxcmZl/rqC/zcAHI6JLZm7IzDkN1h8AfDAzX8vM+Zm5voljnQFcm5nPZuYGSp/9mIhoeNn2ZZn558x8glKg+vAOPhdJ0k4wyEhSsb2X0igMAOUv1GsojUC8F1jZYFsCtY07KG97kNJowg3ACxFxU0S8s8IaDgZ+txM1d6E0enEhMAzoUF7/fuC95cu5XoqIl4BJwIHl7Vu9n0avm1rXXH+fozQCtbR8+dgp5fW3AvcBt0XE8xFxdUR0YFtbffbl1+0b9A9bB8dXy+9bktQCDDKSVGzPU/rCDkBEvIPSaML/A1YB3Rtsi4bLjWXm9Zl5NHAkpS/4F23Z1EwNKyldmlWx8kjHt4CNwD806Oe5zHxXg5/9M/Nj5e1bvR9KAWqbrhvVtd3+MvOZzBwL/BVwFfDTiHhHZm7OzMsysxelS/FOAc5s4lhbffbA+4A64IWd+CgkSW+SQUaSiqNDeSL6lp/2wH8Bn42IoyJiP+BfgMcycwXwP0B1RJxWbvsF4K+b6jgijomIAeWRh1coBYzXyptfoDQHZHu+C1wREYeWbxrQOyIOqPA9fZPSjQQ6AnOB9eUJ+28r30TgQxFxTLnt7cAlEfHuiOgGTGim7x32FxGfjoiumfk68FJ5n9ciYnhEVEdEO2A9pUvNXmui/x8BEyOiR0R0ovTZ/9i7pUnSnmGQkaTiuJfSpPMtP5dm5q+ArwF3UBqx+ADlOSCZ+SIwGria0uVmvYB5wF+a6PudlOaPrKV0idQa4JrytmlAr/LlWT9rYt9rKYWM+yl98Z8GvK3C9/Q/5WOek5mvASMp3cXsOUrPmfkupYn3AJdTujTuOWAGpRsZNPVegNKoTzP9nQQsiogNlCb+j8nMjZTC3k/L72UJMAv4QROH+B6ly9AeLve/ETi/wvctSdpFPhBTkvYS5buC1QJnZOZDrV3ProqI8yiFj+NbuxZJ0p7niIwkvYVFxN9ExLvKl51NAgKY08xubVJEHBQRQ8rPbzkM+BJwV2vXJUlqHe2bbyJJKrBBlObR7AssBk4r35q4iPYFbgR6UJrTchul2zdLkvZCXlomSZIkqXC8tEySJElS4RhkJEmSJBVOq82R6dKlS1ZVVbXW4SVJkiS1cfPnz38xM7s2ta3VgkxVVRXz5s1rrcNLkiRJauMi4vfb2+alZZIkSZIKxyAjSZIkqXAMMpIkSZIKxwdiSpIkqU3ZvHkztbW1bNy4sbVL0R7SsWNHunfvTocOHSrexyAjSZKkNqW2tpb999+fqqoqIqK1y9FulpmsWbOG2tpaevToUfF+zV5aFhHfi4g/RsTT29keEXF9RCyPiCcjou9O1C1JkiRtZePGjRxwwAGGmL1ERHDAAQfs9AhcJXNkpgMn7WD7ycCh5Z9zgf/cqQokSZKkRgwxe5c38/tuNshk5sPAn3bQ5FTgliyZA7wrIg7a6UokSZKkAvnGN77BD3/4w9YuY6fNnDmTRx99dKfbTZ06lVtuuWV3lrZTWmKOTDdgZYPl2vK6VS3QtyRJkvZyUx74bYv2N/GEni3Sz/3338/tt9/eIn3tSTNnzqRTp04MHjx4p9qNHz9+T5RXsZa4/XJT40DZZMOIcyNiXkTMW716dQscWpIkSWpZV199Nddffz0AEydO5CMf+QgAv/rVr/j0pz8NwPr169m0aRNdu3bdat+5c+cyePBg+vTpw+DBg1m2bBkA06dPZ8KECfXtTjnlFGbOnAnAtGnT6NmzJ8OGDeOcc86pbzdu3DjOO+88hg8fziGHHMKsWbM4++yzOeKIIxg3blx9X/fffz+DBg2ib9++jB49mg0bNgBQVVXF17/+dfr27Ut1dTVLly5lxYoVTJ06lSlTpnDUUUcxe/Zs7rnnHgYMGECfPn0YMWIEL7zwQpPtLr30Uq655hoAampqGDhwIL1792bUqFGsXbsWgGHDhnHxxRfTv39/evbsyezZs1vyV7OVlggytcDBDZa7A8831TAzb8rMfpnZr/EvXZIkSWoLjjvuuPov4PPmzWPDhg1s3ryZX//61wwdOhSAGTNm8NGPfnSbfQ8//HAefvhhFi5cyOWXX86kSZN2eKznn3+eK664gjlz5vDAAw+wdOnSrbavXbuWBx98kClTpjBy5EgmTpzIokWLeOqpp6ipqeHFF1/kyiuvZMaMGSxYsIB+/fpx7bXX1u/fpUsXFixYwHnnncc111xDVVUV48ePZ+LEidTU1DB06FCOPfZY5syZw8KFCxkzZgxXX311k+0aOvPMM7nqqqt48sknqa6u5rLLLqvfVldXx9y5c7nuuuu2Wt/SWuLSsruBCRFxGzAAWJeZhbusrKWHLHeHie3vaO0SKjP8ktauQJIk6U07+uijmT9/Pi+//DL77bcfffv2Zd68ecyePbt+pOaXv/wln/3sZ7fZd926dZx11lk888wzRASbN2/e4bHmzp3L8ccfz3ve8x4ARo8ezW9/+8b30pEjRxIRVFdXc+CBB1JdXQ3AkUceyYoVK6itrWXx4sUMGTIEgE2bNjFo0KD6/T/xiU/Uv6c777yzyRpqa2s5/fTTWbVqFZs2bWr2Fsjr1q3jpZde4vjjjwfgrLPOYvTo0U0ec8WKFTvsa1dUcvvlHwG/AQ6LiNqI+FxEjI+ILRfJ3Qs8CywHvgP8w26rVpIkSdrNOnToQFVVFd///vcZPHgwQ4cO5aGHHuJ3v/sdRxxxBFAKIP37999m36997WsMHz6cp59+mnvuuaf+lsLt27fn9ddfr2+3ZX1mkzMy6u23334A7LPPPvWvtyzX1dWRmZxwwgnU1NRQU1PD4sWLmTZt2jb7t2vXjrq6uiaPcf755zNhwgSeeuopbrzxxl1+EGklx2wJldy1bGxmHpSZHTKze2ZOy8ypmTm1vD0z8wuZ+YHMrM7MebutWkmSJGkPOO6447jmmms47rjjGDp0KFOnTuWoo44iIli0aBGHH3447dq122a/devW0a1bN6A0L2aLqqoqampqeP3111m5ciVz584FoH///syaNYu1a9dSV1fHHXfs3BU4AwcO5JFHHmH58uUAvPrqq1uN6DRl//335+WXX26y5ptvvnm77bbo3Lkz7373u+svv7v11lvrR2f2pJaYIyNJkiS9pQwdOpRVq1YxaNAgDjzwQDp27Fg/T+QXv/gFJ53U9GMWv/zlL3PJJZcwZMgQXnvttfr1Q4YMoUePHlRXV3PhhRfSt2/pGfLdunVj0qRJDBgwgBEjRtCrVy86d+5ccZ1du3Zl+vTpjB07lt69ezNw4MBt5tk0NnLkSO66666tJvGPHj2aoUOH0qVLl+22a+jmm2/moosuonfv3tTU1DB58uSKa24p0dxw1u7Sr1+/nDev7QzeOEemBTlHRpIk7YIlS5bUX8LVFp1wwgnccsstHHRQyzw6ccOGDXTq1Im6ujpGjRrF2WefzahRo1qk7yJp6vceEfMzs19T7Vtisr8kSZK013jggQdatL9LL72UGTNmsHHjRk488UROO+20Fu3/rcogI0mSJLWiLc9m0c5xjowkSZKkwjHISJIkSSocg4wkSZKkwnGOjKRdVoS7/gFMPKFna5cgSZJaiEFGkqRW5h8DpF3zwvpdexL9m3X9t/6Vbgd35+8+Nbai9ge+s+NurmjXzJw5k3333ZfBgwfvVLupU6fy9re/nTPPPHNPlFnPICNJkqS27aFv7HDzO/5St1PdvTL4ol2ppt7Mh2Zw0/QftEhfbcHMmTPp1KlTRUGmYbvx48fvifK24RwZSZIkqYH/uO5bfHfqDQBMvuQi/u6UkwCYPfMhvnDOZwF4ef16Nm/aRJcuXVn5h9/zyZEnM3zwMXxy5MnUrvzDNn0umP84gwcPpk+fPgwePJhly5YBMH36dCZMmFDf7pRTTmHmzJkATJs2jZ49ezJs2DDOOeec+nbjxo3jvPPOY/jw4RxyyCHMmjWLs88+myOOOIJx48bV93X//fczaNAg+vbty+jRo9mwYQMAVVVVfP3rX6dv375UV1ezdOlSVqxYwdSpU5kyZQpHHXUUs2fP5p577mHAgAH06dOHESNG8MILLzTZ7tJLL62/hXRNTQ0DBw6kd+/ejBo1irVr1wIwbNgwLr74Yvr370/Pnj2ZPXv2Lv+eDDKSJElSA4OGHMucRx8B4ImFC3jllQ1s3ryZx+Y8yoBBQwB4eOaDHHv8cAAmXXQBo8eewUOPPs4nPjWGr178pW36PPTQw3j44YdZuHAhl19+OZMmTdphDc8//zxXXHEFc+bM4YEHHmDp0qVbbV+7di0PPvggU6ZMYeTIkUycOJFFixbx1FNPUVNTw4svvsiVV17JjBkzWLBgAf369ePaa6+t379Lly4sWLCA8847j2uuuYaqqirGjx/PxIkTqampYejQoRx77LHMmTOHhQsXMmbMGK6++uom2zV05plnctVVV/Hkk09SXV3NZZddVr+trq6OuXPnct111221/s3y0jJJkiSpgd5H9eXJmoVsePll9t13P6o/fBRPLJzPY48+wpVXfwuAh2Y8wJhPfwaA+XMf43s/uA2A0WP+nisn/9M2fa5fv47R5/8fnnnmGSKCzZs377CGuXPncvzxx/Oe97yn1O/o0fz2t2/Mpxs5ciQRQXV1NQceeCDV1dUAHHnkkaxYsYLa2loWL17MkCGl4LVp0yYGDRpUv/8nPvEJAI4++mjuvPPOJmuora3l9NNPZ9WqVWzatIkePXrssOZ169bx0ksvcfzxxwNw1llnMXr06CaPuWLFih32VQlHZCRJkqQGOnTowMHvez+3/fAW+g0YyIBBQ3jk4YdZ8dyz9DzscAAWLphHn6OPabqDiG1WXfXPlzN8+HCefvpp7rnnHjZuLN2goH379rz++uv17basz8wd1rjffvsBsM8++9S/3rJcV1dHZnLCCSdQU1NDTU0NixcvZtq0advs365dO+rqmp5jdP755zNhwgSeeuopbrzxxvra3qxKjrkzDDKSJElSIwOHHMt//vt1DBp8LAMHD+GW73+HD/XuTUSwdMliPnhoT9q1awdAvwED+dkdPwHgjttvY0CDkY8tXl6/jm7dugGleTFbVFVVUVNTw+uvv87KlSuZO3cuAP3792fWrFmsXbuWuro67rjjjp2rf+BAHnnkEZYvXw7Aq6++utWITlP2339/Xn755frldeveqPnmm2/ebrstOnfuzLvf/e76+S+33npr/ejM7mCQkSRJkhoZOGgIL/zv/3J0/wF0/asD2W+/jvXzYx584D6Gjzixvu2VV32L235wC8MHH8NPb/svrvjmNdv094V/vIBLLrmEIUOG8Nprr9WvHzJkCD169KC6upoLL7yQvn37AtCtWzcmTZrEgAEDGDFiBL169aJz584V19+1a1emT5/O2LFj6d27NwMHDtxmnk1jI0eO5K677tpqEv/o0aMZOnQoXbp02W67hm6++WYuuugievfuTU1NDZMnT6645p0VzQ1b7S79+vXLefPmtcqxm1KEe/hPbL9zSbzVDL+ktSvQHlaE8wd8BobaLs8haWtLlizhiCOOqLj9nn6OzKdO/Vv+/cbvcuBfH7RT++3sc2Q2bNhAp06dqKurY9SoUZx99tmMGjVqp/ookqZ+7xExPzP7NdXeyf6S9h7NPIegzfCPAZLUpt3+3/+zR45z6aWXMmPGDDZu3MiJJ57IaaedtkeOWxQGGUmSJKkN2vJsFjXNOTKSJEmSCscRGUmSVBkvz9QelJlEE7cxLrT1q1q7gsq8c+fm/rSENzNv3xEZSZIktSkdO3ZkzZo1b+rLrYonM1mzZg0dO+7czRAckZEkSVKb0r17d2pra1m9enVF7df/efNurqhl/Clebe0SKtPxpT1/yI4d6d69+07tY5CRJElSm9KhQwd69OhRcfvC3MK8/Z6529kuK8jlmV5aJkmSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwKgoyEXFSRCyLiOUR8ZUmtr8vIh6KiIUR8WREfKzlS5UkSZKkkmaDTES0A24ATgZ6AWMjolejZl8Fbs/MPsAY4NstXagkSZIkbVHJiEx/YHlmPpuZm4DbgFMbtUngneXXnYHnW65ESZIkSdpa+wradANWNliuBQY0anMpcH9EnA+8AxjRItVJkiRJUhMqGZGJJtZlo+WxwPTM7A58DLg1IrbpOyLOjYh5ETFv9erVO1+tJEmSJFFZkKkFDm6w3J1tLx37HHA7QGb+BugIdGncUWbelJn9MrNf165d31zFkiRJkvZ6lQSZx4FDI6JHROxLaTL/3Y3a/AH4KEBEHEEpyDjkIkmSJGm3aDbIZGYdMAG4D1hC6e5kiyLi8oj4eLnZl4BzIuIJ4EfAuMxsfPmZJEmSJLWISib7k5n3Avc2Wje5wevFwJCWLU2SJEmSmlbRAzElSZIkqS0xyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqnIqCTEScFBHLImJ5RHxlO20+FRGLI2JRRPxXy5YpSZIkSW9o31yDiGgH3ACcANQCj0fE3Zm5uEGbQ4FLgCGZuTYi/mp3FSxJkiRJlYzI9AeWZ+azmbkJuA04tVGbc4AbMnMtQGb+sWXLlCRJkqQ3VBJkugErGyzXltc11BPoGRGPRMSciDipqY4i4tyImBcR81avXv3mKpYkSZK016skyEQT67LRcnvgUGAYMBb4bkS8a5udMm/KzH6Z2a9r1647W6skSZIkAZUFmVrg4AbL3YHnm2jz35m5OTOfA5ZRCjaSJEmS1OIqCTKPA4dGRI+I2BcYA9zdqM3PgOEAEdGF0qVmz7ZkoZIkSZK0RbNBJjPrgAnAfcAS4PbMXBQRl0fEx8vN7gPWRMRi4CHgosxcs7uKliRJkrR3a/b2ywCZeS9wb6N1kxu8TuCC8o8kSZIk7VYVPRBTkiRJktoSg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSqcioJMRJwUEcsiYnlEfGUH7T4ZERkR/VquREmSJEnaWrNBJiLaATcAJwO9gLER0auJdvsDXwQea+kiJUmSJKmhSkZk+gPLM/PZzNwE3Aac2kS7K4CrgY0tWJ8kSZIkbaOSINMNWNlguba8rl5E9AEOzsyft2BtkiRJktSkSoJMNLEu6zdG7ANMAb7UbEcR50bEvIiYt3r16sqrlCRJkqQGKgkytcDBDZa7A883WN4f+BAwMyJWAAOBu5ua8J+ZN2Vmv8zs17Vr1zdftSRJkqS9WiVB5nHg0IjoERH7AmOAu7dszMx1mdklM6syswqYA3w8M+ftloolSZIk7fWaDTKZWQdMAO4DlgC3Z+aiiLg8Ij6+uwuUJEmSpMbaV9IoM+8F7m20bvJ22g7b9bIkSZIkafsqeiCmJEmSJLUlBhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4FQWZiDgpIpZFxPKI+EoT2y+IiMUR8WRE/Coi3t/ypUqSJElSSbNBJiLaATcAJwO9gLER0atRs4VAv8zsDfwUuLqlC5UkSZKkLSoZkekPLM/MZzNzE3AbcGrDBpn5UGa+Wl6cA3Rv2TIlSZIk6Q2VBJluwMoGy7XlddvzOeAXu1KUJEmSJO1I+wraRBPrssmGEZ8G+gHHb2f7ucC5AO973/sqLFGSJEmStlbJiEwtcHCD5e7A840bRcQI4J+Aj2fmX5rqKDNvysx+mdmva9eub6ZeSZIkSaooyDwOHBoRPSJiX2AMcHfDBhHRB7iRUoj5Y8uXKUmSJElvaDbIZGYdMAG4D1gC3J6ZiyLi8oj4eLnZvwKdgJ9ERE1E3L2d7iRJkiRpl1UyR4bMvBe4t9G6yQ1ej2jhuiRJkiRpuyp6IKYkSZIktSUGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgVBZmIOCkilkXE8oj4ShPb94uIH5e3PxYRVS1dqCRJkiRt0WyQiYh2wA3AyUAvYGxE9GrU7HPA2sz8IDAFuKqlC5UkSZKkLSoZkekPLM/MZzNzE3AbcGqjNqcCN5df/xT4aEREy5UpSZIkSW+oJMh0A1Y2WK4tr2uyTWbWAeuAA1qiQEmSJElqLDJzxw0iRgN/k5mfLy9/Buifmec3aLOo3Ka2vPy7cps1jfo6Fzi3vHgYsKyl3sheogvwYmsXIRWY55C0azyHpF3jObTz3p+ZXZva0L6CnWuBgxssdwee306b2ohoD8mplQ0AAAUXSURBVHQG/tS4o8y8Cbipkoq1rYiYl5n9WrsOqag8h6Rd4zkk7RrPoZZVyaVljwOHRkSPiNgXGAPc3ajN3cBZ5defBB7M5oZ6JEmSJOlNanZEJjPrImICcB/QDvheZi6KiMuBeZl5NzANuDUillMaiRmzO4uWJEmStHer5NIyMvNe4N5G6yY3eL0RGN2ypakJXpYn7RrPIWnXeA5Ju8ZzqAU1O9lfkiRJktqaSubISJIkSVKbYpCRtNeIiEeb2X5vRLxrT9UjvdVERFVEPF1+PSwift7aNUmtKSK+GBFLIuKOiPhNRPwlIi5s7breKiqaIyNJbU1EtMvM13Zmn8wc3Mz2j+1aVVIxRURQutz89dauRXqL+QfgZOAV4P3Aaa1bzluLIzJtTET8LCLmR8Si8gNEiYgNDbZ/MiKml18fGBF3RcQT5Z8dfkmTiqL8V92lEXFzRDwZET+NiLdHxIqImBwRvwZGR8QHIuKX5XNmdkQcXt6/yXNjy7kUEQdFxMMRURMRT0fE0PL6FRHRpfz6gvK2pyPi/zaoa0lEfKd8jt4fEW9rlQ9J2kUN/j1/G1gAfKb8F+MFEfGTiOhUbndMRDxaPpfmRsT+5X1nl9su8P8/0rYiYipwCKXHlJyRmY8Dm1u3qrcWR2TanrMz80/lL0ePR8QdO2h7PTArM0dFRDug054pUdojDgM+l5mPRMT3KP1VC2BjZh4LEBG/AsZn5jMRMQD4NvARmj83/h64LzP/ubz97Q03RsTRwGeBAUAAj0XELGAtcCgwNjPPiYjbgb8DftDi717aMw6j9G99MnAnMCIzX4mIi4ELIuKbwI+B0zPz8Yh4J/Bn4I/ACZm5MSIOBX4E+JA/qYHMHB8RJwHDM/PF1q7nrcgg0/Z8MSJGlV8fTOlL0/Z8BDgToHyJzbrdXJu0J63MzEfKr38AfLH8+scA5b8WDwZ+UroqBoD9yv9t7tx4HPheRHQAfpaZNY22HwvclZmvlI91JzCU0l/VnmvQfj5QtQvvUWptv8/MORFxCtALeKR8Pu0L/IZS0FlV/ksymbkeICLeAfxHRBwFvAb0bI3iJe3dDDJtSEQMA0YAgzLz1YiYCXQEGt4ju2MrlCa1hsb3ht+y/Er5v/sAL2XmUTvdcebDEXEc8LeUHub7r5l5S4MmsZ1dAf7S4PVrgJeWqci2nE8BPJCZYxtujIjebHsuAkwEXgA+TOlc3Lg7i5SkpjhHpm3pDKwth5jDgYHl9S9ExBERsQ8wqkH7XwHnQWnic3nIX3qreF9EDCq/Hgv8uuHG8l+Gn4uI0VCarBwRHy5v3uG5ERHvB/6Ymd8BpgF9Gx37YeC08rycd1A672a33FuT2pw5wJCI+CBA+d9+T2Ap8N6IOKa8fv+IaE/p/1eryjcH+AzQrpXqlrQXM8i0Lb8E2kfEk8AVlP7HAvAV4OfAg8CqBu3/ERgeEU9RusTlyD1Yq7S7LQHOKp8P7wH+s4k2ZwCfi4gngEXAqeX1zZ0bw4CaiFhIaY7LvzXcmJkLgOnAXOAx4LuZubAF3pPUJmXmamAc8KPyOTcHODwzNwGnA/9ePs8eoHRlwLcpnZ9zKF1W9kqTHUsCICL+OiJqgQuAr0ZErX+A3nWR2dSIsSS1noioAn6emR9q5VIkSVIb5YiMJEmSpMJxREaSJElS4TgiI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCuf/AzILQ7KWCxj9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1008x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PERFORMANCE ON THE FAKE TEST ###\n",
    "\n",
    "plt.figure(figsize=(14,5))\n",
    "plt.bar(np.arange(len(logistic_train_aug_new))-0.1, logistic_train_aug_new.values(), \n",
    "        alpha=0.5, width=0.2, label='w/ augmentation')\n",
    "plt.bar(np.arange(len(logistic_train_aug_new))+0.1, logistic_train_new.values(), \n",
    "        alpha=0.5, width=0.2, label='w/o augmentation')\n",
    "plt.xticks(range(len(logistic_train_aug_new)), logistic_train_aug_new.keys())\n",
    "plt.title('Logistic Regression'); plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAE/CAYAAACdNPbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df5yWdZ3v8ddHINgNlkpY10Ab2pUAg4CInw6CQGutqLSRsqmoHT2wBz2H1izdjfzRrqvLI8xz6lC7JOBa5qZ2tIc/ABVEjRBhUkFMKoxZWFNDBI1g5HP+mJvxdhjgRgZmLnk9H495eF3X93td1+dmuGTe8/1+7zsyE0mSJEkqkqNaugBJkiRJOlAGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJB20iBgVEbUtXYck6chhkJGkd6mIWB8Rv4+IbRHxXxExNyI6tnRdBysiMiJeL72ubRHx6mG+v6FNkloBg4wkvbuNz8yOQH9gAHBFC9fTXD6WmR1LX+870JMjou2hKEqSdPgYZCTpCJCZ/wU8QH2gASAi/ioiVkXEaxGxISKuKmurKo18TI6I30TEyxHx92Xtf1Qa4dkcEWuAT5TfLyJ6R8TiiHg1IlZHxOllbXMj4tsRcV9pROWxiPiziLixdL21ETHgnbzOiLgoItZFxO8i4u6I+GBZW0bE/4iI54HnS8d6RcTCUv/nIuJzZf0/HRFrImJrRPxnRFwWEe8F7gM+WDYi9ME9CpEkHXIGGUk6AkREd+BTwLqyw68D5wHvA/4KmBoRZzY69STgI8AYYEZE9C4d/xrw56WvvwQml92rHXAPsAD4U+AS4NaI+EjZdT8H/APQBfgD8FNgZWn/R8A33sFrPAW4rnTtY4EXgNsadTsTGAL0KYWShcD3S3VOAr4dESeW+s4B/ntmdgI+CjyUma9T/+e4sWxEaOOB1ipJOngGGUl6d/txRGwFNgC/pT6AAJCZizPz6czclZlPAT8ATm50/tWZ+fvM/Dnwc+BjpeOfA/4xM3+XmRuAm8rOGQp0BP45M3dk5kPAT6gPCrvdlZlPZuZ24C5ge2bOz8w3gR9SPw1uX1aWRntejYjd9/488L3MXJmZf6B+Gt2wiKgqO++6Us2/B04D1mfmzZlZl5krgTuAz5b67qQ+8PxJZm4utUuSWgmDjCS9u51ZGlEYBfSifsQDgIgYEhEPR8RLEbEFmFLeXvJfZdtvUB9QAD5IfTja7YWy7Q8CGzJzV6P2bmX7L5Zt/76J/f29KcHAzHxf6evSsvs21JGZ24BXGt23vOYPAUPKAtGr1IehPyu1/zXwaeCFiFgSEcP2U5Mk6TAyyEjSESAzlwBzgZllh78P3A0cl5mdgdlAVHjJTcBxZfvHl21vBI6LiKMatf/nAZZ9oDZSH04AKE0dO7rRfbNsewOwpCwQva80VWwqQGY+kZlnUD/t7MfA7U1cQ5LUQgwyknTkuBEYFxG7F/x3An6XmdsjYjDwNwdwrduBKyLi/aX1N5eUtf2M+vU3l0dEu4gYBYxnz/Uqze37wAUR0T8i2gP/BPwsM9fvpf9PgJ4RcW6pznYR8YnSGxW8JyI+HxGdM3Mn8BrwZum8F4GjI6LzIX49kqR9MMhI0hEiM18C5gNfLR36W+Ca0hqaGbw14lCJq6mfxvVr6hf131J2nx3A6dQvin8Z+DZwXmauPdjXsC+Z+SD1r+0O6keM/hw4ex/9twKfLPXZSP00uuuB9qUu5wLrI+I16qfdnVM6by3164l+VZqS5ruWSVILiExHyCVJkiQViyMykiRJkgrHICNJkiSpcAwykiRJkgrHICNJkiSpcAwykiRJkgqnbUvduEuXLllVVdVSt5ckSZLUyj355JMvZ2bXptpaLMhUVVWxYsWKlrq9JEmSpFYuIl7YW5tTyyRJkiQVjkFGkiRJUuEYZCRJkiQVToutkZEkSZKasnPnTmpra9m+fXtLl6LDpEOHDnTv3p127dpVfI5BRpIkSa1KbW0tnTp1oqqqioho6XJ0iGUmr7zyCrW1tfTo0aPi85xaJkmSpFZl+/btHH300YaYI0REcPTRRx/wCJxBRpIkSa2OIebI8k6+3wYZSZIk6R247rrruPXWW1u6jAO2ePFiHn/88QPuN3v2bObPn38oSzsgrpGRJElSqzZr4S+a9XrTx/VslussWLCA22+/vVmudTgtXryYjh07Mnz48APqN2XKlMNRXsUckZEkSZLK3HDDDdx0000ATJ8+nVNOOQWABx98kHPOOQeA1157jR07dtC1a9e3nbt8+XKGDx/OgAEDGD58OM899xwAc+fOZdq0aQ39TjvtNBYvXgzAnDlz6NmzJ6NGjeKiiy5q6Hf++eczdepURo8ezYc//GGWLFnChRdeSO/evTn//PMbrrVgwQKGDRvGwIEDmThxItu2bQOgqqqKr33tawwcOJC+ffuydu1a1q9fz+zZs5k1axb9+/dn6dKl3HPPPQwZMoQBAwYwduxYXnzxxSb7XXXVVcycOROAmpoahg4dSr9+/ZgwYQKbN28GYNSoUXz5y19m8ODB9OzZk6VLlzbnt+ZtDDKSJElSmZEjRzb8AL5ixQq2bdvGzp07efTRR6murgZg0aJFjBkzZo9ze/XqxSOPPMKqVau45ppruPLKK/d5r40bN3LttdeybNkyFi5cyNq1a9/WvnnzZh566CFmzZrF+PHjmT59OqtXr+bpp5+mpqaGl19+ma9//essWrSIlStXMmjQIL7xjW80nN+lSxdWrlzJ1KlTmTlzJlVVVUyZMoXp06dTU1NDdXU1J510EsuWLWPVqlWcffbZ3HDDDU32K3feeedx/fXX89RTT9G3b1+uvvrqhra6ujqWL1/OjTfe+Lbjzc2pZZKOHA9f19IVVGb0FS1dgSQd0T7+8Y/z5JNPsnXrVtq3b8/AgQNZsWIFS5cubRipuf/++7ngggv2OHfLli1MnjyZ559/nohg586d+7zX8uXLOfnkk/nABz4AwMSJE/nFL96aSjd+/Hgigr59+3LMMcfQt29fAE488UTWr19PbW0ta9asYcSIEQDs2LGDYcOGNZz/mc98puE13XnnnU3WUFtby1lnncWmTZvYsWPHft8CecuWLbz66qucfPLJAEyePJmJEyc2ec/169fv81oHwxEZSZIkqUy7du2oqqri5ptvZvjw4VRXV/Pwww/zy1/+kt69ewP1AWTw4MF7nPvVr36V0aNH88wzz3DPPfc0vKVw27Zt2bVrV0O/3cczc5+1tG/fHoCjjjqqYXv3fl1dHZnJuHHjqKmpoaamhjVr1jBnzpw9zm/Tpg11dXVN3uOSSy5h2rRpPP3003znO9856A8ireSezcEgI0mSJDUycuRIZs6cyciRI6murmb27Nn079+fiGD16tX06tWLNm3a7HHeli1b6NatG1C/Lma3qqoqampq2LVrFxs2bGD58uUADB48mCVLlrB582bq6uq44447DqjOoUOH8thjj7Fu3ToA3njjjbeN6DSlU6dObN26tcma582bt9d+u3Xu3Jn3v//9DdPvbrnllobRmcPJICNJkiQ1Ul1dzaZNmxg2bBjHHHMMHTp0aFgnct9993Hqqac2ed7ll1/OFVdcwYgRI3jzzTcbjo8YMYIePXrQt29fLrvsMgYOHAhAt27duPLKKxkyZAhjx46lT58+dO7cueI6u3btyty5c5k0aRL9+vVj6NChe6yzaWz8+PHcddddb1vEP3HiRKqrq+nSpcte+5WbN28eX/rSl+jXrx81NTXMmDGj4pqbS+xvOOtQGTRoUK5YsaJF7t2U5n5bv0Ohud4qUDpiuUZGkgrh2WefbZjC1RqNGzeO+fPnc+yxxzbL9bZt20bHjh2pq6tjwoQJXHjhhUyYMKFZrl0kTX3fI+LJzBzUVH8X+0uSJEkHYOHChc16vauuuopFixaxfft2PvnJT3LmmWc26/XfrQwykg5aEUY0Aab7fzxJUiu0+7NZdGBcIyNJkiSpcPz9ZJE4v1+SJEkCHJGRJEmSVEAGGUmSJEmFY5CRJEmS3oHrrruOW2+9taXLaDaLFy/m8ccfP+B+s2fPZv78+YeytCa5RkaSJEmtW3OvE26m9bwLFizg9ttvb5ZrtQaLFy+mY8eODB8+/ID6TZky5XCUtwdHZCRJkqQyN9xwAzfddBMA06dP55RTTgHgwQcf5JxzzgHgtddeY8eOHXTt2pUXXniBMWPG0K9fP8aMGcNvfvObPa65fPlyhg8fzoABAxg+fDjPPfccAHPnzmXatGkN/U477TQWL14MwJw5c+jZsyejRo3ioosuauh3/vnnM3XqVEaPHs2HP/xhlixZwoUXXkjv3r05//zzG661YMEChg0bxsCBA5k4cSLbtm0DoKqqiq997WsMHDiQvn37snbtWtavX8/s2bOZNWsW/fv3Z+nSpdxzzz0MGTKEAQMGMHbsWF588cUm+1111VUNbyFdU1PD0KFD6devHxMmTGDz5s0AjBo1ii9/+csMHjyYnj17snTp0oP+PhlkJEmSpDIjR45s+EF7xYoVbNu2jZ07d/Loo49SXV0NwKJFixgzZgwA06ZN47zzzuOpp57i85//PJdeeuke1+zVqxePPPIIq1at4pprruHKK6/cZw0bN27k2muvZdmyZSxcuJC1a9e+rX3z5s089NBDzJo1i/HjxzN9+nRWr17N008/TU1NDS+//DJf//rXWbRoEStXrmTQoEF84xvfaDi/S5curFy5kqlTpzJz5kyqqqqYMmUK06dPp6amhurqak466SSWLVvGqlWrOPvss7nhhhua7FfuvPPO4/rrr+epp56ib9++XH311Q1tdXV1LF++nBtvvPFtx98pp5ZJkiRJZT7+8Y/z5JNPsnXrVtq3b8/AgQNZsWIFS5cubRipuf/++7ngggsA+OlPf8qdd94JwLnnnsvll1++xzW3bNnC5MmTef7554kIdu7cuc8ali9fzsknn8wHPvABACZOnMgvfvHWB1CPHz+eiKBv374cc8wx9O3bF4ATTzyR9evXU1tby5o1axgxYgQAO3bsYNiwYQ3nf+Yzn2l4rbtrb6y2tpazzjqLTZs2sWPHDnr06LHPmrds2cKrr77KySefDMDkyZOZOHFik/dcv379Pq9VCUdkJEmSpDLt2rWjqqqKm2++meHDh1NdXc3DDz/ML3/5S3r37g3UB43Bgwc3eX5E7HHsq1/9KqNHj+aZZ57hnnvuYfv27QC0bduWXbt2NfTbfTwz91lj+/btATjqqKMatnfv19XVkZmMGzeOmpoaampqWLNmDXPmzNnj/DZt2lBXV9fkPS655BKmTZvG008/zXe+852G2t6pSu55IAwykiRJUiMjR45k5syZjBw5kurqambPnk3//v2JCFavXk2vXr1o06YNAMOHD+e2224D4NZbb+Wkk07a43pbtmyhW7duQP26mN2qqqqoqalh165dbNiwgeXLlwMwePBglixZwubNm6mrq+OOO+44oPqHDh3KY489xrp16wB444033jai05ROnTqxdevWJmueN2/eXvvt1rlzZ97//vc3TMu75ZZbGkZnDgWDjCRJktRIdXU1mzZtYtiwYRxzzDF06NChYT3Ifffdx6mnntrQ96abbuLmm2+mX79+3HLLLXzzm9/c43qXX345V1xxBSNGjODNN99sOD5ixAh69OhB3759ueyyyxg4cCAA3bp148orr2TIkCGMHTuWPn360Llz54rr79q1K3PnzmXSpEn069ePoUOH7rHOprHx48dz1113vW0R/8SJE6murqZLly577Vdu3rx5fOlLX6Jfv37U1NQwY8aMims+ULG/YatDZdCgQblixYoWuXdTZi3cd0JtDaa3PbAk3mKa6S0NVRxFeH7AZ0iSiuLZZ59tmMLVGo0bN4758+dz7LHHHtL7bNu2jY4dO1JXV8eECRO48MILmTBhwiG9Z0tq6vseEU9m5qCm+rvYX5IkSToACxcuPCz3ueqqq1i0aBHbt2/nk5/8JGeeeeZhuW9RGGQkSZKkVmj3Z7Ooaa6RkSRJklQ4BhlJkiS1Oi21jlst4518vw0ykiRJalU6dOjAK6+8Ypg5QmQmr7zyCh06dDig81wjI0mSpFale/fu1NbW8tJLL7V0KTpMOnToQPfu3Q/onIqCTEScCnwTaAP8W2b+c6P244F5wPtKfb6SmfceUCWSJEkS0K5dO3r06NHSZaiV2+/UsohoA3wL+BTQB5gUEX0adfsH4PbMHACcDXy7uQuVJEmSpN0qWSMzGFiXmb/KzB3AbcAZjfok8Cel7c7AxuYrUZIkSZLerpKpZd2ADWX7tcCQRn2uAhZExCXAe4GxTV0oIi4GLgY4/vjjD7RWSZIkaQ+zFv6ipUuoyPRxPVu6hHeVSkZkooljjd9CYhIwNzO7A58GbomIPa6dmd/NzEGZOahr164HXq0kSZIkUVmQqQWOK9vvzp5Tx74A3A6QmT8FOgBdmqNASZIkSWqskiDzBHBCRPSIiPdQv5j/7kZ9fgOMAYiI3tQHGd8vT5IkSdIhsd8gk5l1wDTgAeBZ6t+dbHVEXBMRp5e6/R1wUUT8HPgBcH76CUaSJEmSDpGKPkem9Jkw9zY6NqNsew0wonlLkyRJkqSmVTK1TJIkSZJaFYOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMKp6O2XJUnSoTNr4S9auoSKTB/Xs6VLkKQGjshIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCMchIkiRJKhyDjCRJkqTCadvSBUiSJElHhIeva+kKKjP6ipauoCKOyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMJp29IFSJKkgnj4upauoDKjr2jpCiQdBo7ISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSqcioJMRJwaEc9FxLqI+Mpe+nwuItZExOqI+H7zlilJkiRJb2m7vw4R0Qb4FjAOqAWeiIi7M3NNWZ8TgCuAEZm5OSL+9FAVLEmSJEmVjMgMBtZl5q8ycwdwG3BGoz4XAd/KzM0Amfnb5i1TkiRJkt5SSZDpBmwo268tHSvXE+gZEY9FxLKIOLW5CpQkSZKkxvY7tQyIJo5lE9c5ARgFdAeWRsRHM/PVt10o4mLgYoDjjz/+gIuVJEmSJKhsRKYWOK5svzuwsYk+/y8zd2bmr4HnqA82b5OZ383MQZk5qGvXru+0ZkmSJElHuEqCzBPACRHRIyLeA5wN3N2oz4+B0QAR0YX6qWa/as5CJUmSJGm3/QaZzKwDpgEPAM8Ct2fm6oi4JiJOL3V7AHglItYADwNfysxXDlXRkiRJko5slayRITPvBe5tdGxG2XYCXyx9SZIkSdIhVdEHYkqSJElSa2KQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4FQWZiDg1Ip6LiHUR8ZV99PtsRGREDGq+EiVJkiTp7fYbZCKiDfAt4FNAH2BSRPRpol8n4FLgZ81dpCRJkiSVq2REZjCwLjN/lZk7gNuAM5rody1wA7C9GeuTJEmSpD1UEmS6ARvK9mtLxxpExADguMz8STPWJkmSJElNqiTIRBPHsqEx4ihgFvB3+71QxMURsSIiVrz00kuVVylJkiRJZSoJMrXAcWX73YGNZfudgI8CiyNiPTAUuLupBf+Z+d3MHJSZg7p27frOq5YkSZJ0RKskyDwBnBARPSLiPcDZwN27GzNzS2Z2ycyqzKwClgGnZ+aKQ1KxJEmSpCPefoNMZtYB04AHgGeB2zNzdURcExGnH+oCJUmSJKmxtpV0ysx7gXsbHZuxl76jDr4sSZIkSdq7ij4QU5IkSZJaE4OMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqHIOMJEmSpMIxyEiSJEkqnIqCTEScGhHPRcS6iPhKE+1fjIg1EfFURDwYER9q/lIlSZIkqd5+g0xEtAG+BXwK6ANMiog+jbqtAgZlZj/gR8ANzV2oJEmSJO1WyYjMYGBdZv4qM3cAtwFnlHfIzIcz843S7jKge/OWKUmSJElvqSTIdAM2lO3Xlo7tzReA+5pqiIiLI2JFRKx46aWXKq9SkiRJkspUEmSiiWPZZMeIc4BBwL801Z6Z383MQZk5qGvXrpVXKUmSJEll2lbQpxY4rmy/O7CxcaeIGAv8PXByZv6hecqTJEmSpD1VMiLzBHBCRPSIiPcAZwN3l3eIiAHAd4DTM/O3zV+mJEmSJL1lv0EmM+uAacADwLPA7Zm5OiKuiYjTS93+BegI/EdE1ETE3Xu5nCRJkiQdtEqmlpGZ9wL3Njo2o2x7bDPXJUmSJEl7VdEHYkqSJElSa2KQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4FQWZiDg1Ip6LiHUR8ZUm2ttHxA9L7T+LiKrmLlSSJEmSdttvkImINsC3gE8BfYBJEdGnUbcvAJsz8y+AWcD1zV2oJEmSJO1WyYjMYGBdZv4qM3cAtwFnNOpzBjCvtP0jYExERPOVKUmSJElvqSTIdAM2lO3Xlo412Scz64AtwNHNUaAkSZIkNRaZue8OEROBv8zM/1baPxcYnJmXlPVZXepTW9r/ZanPK42udTFwcWn3I8BzzfVCjhBdgJdbugipwHyGpIPjMyQdHJ+hA/ehzOzaVEPbCk6uBY4r2+8ObNxLn9qIaAt0Bn7X+EKZ+V3gu5VUrD1FxIrMHNTSdUhF5TMkHRyfIeng+Aw1r0qmlj0BnBARPSLiPcDZwN2N+twNTC5tfxZ4KPc31CNJkiRJ79B+R2Qysy4ipgEPAG2A72Xm6oi4BliRmXcDc4BbImId9SMxZx/KoiVJkiQd2SqZWkZm3gvc2+jYjLLt7cDE5i1NTXBannRwfIakg+MzJB0cn6FmtN/F/pIkSZLU2lSyRkaSJEmSWhWDjKQjRkQ8vp/2eyPifYerHundJiKqIuKZ0vaoiPhJS9cktaSIuDQino2IOyLipxHxh4i4rKXrereoaI2MJLU2EdEmM988kHMyc/h+2j99cFVJxRQRQf10810tXYv0LvO3wKeA14EPAWe2bDnvLo7ItDIR8eOIeDIiVpc+QJSI2FbW/tmImFvaPiYi7oqIn5e+9vlDmlQUpd/qro2IeRHxVET8KCL+OCLWR8SMiHgUmBgRfx4R95eemaUR0at0fpPPxu5nKSKOjYhHIqImIp6JiOrS8fUR0aW0/cVS2zMR8b/K6no2Iv619IwuiIg/apE/JOkglf19/jawEji39BvjlRHxHxHRsdTvExHxeOlZWh4RnUrnLi31Xem/P9KeImI28GHqP6bk85n5BLCzZat6d3FEpvW5MDN/V/rh6ImIuGMffW8ClmTmhIhoA3Q8PCVKh8VHgC9k5mMR8T3qf6sFsD0zTwKIiAeBKZn5fEQMAb4NnML+n42/AR7IzH8stf9xeWNEfBy4ABgCBPCziFgCbAZOACZl5kURcTvw18C/N/urlw6Pj1D/d30GcCcwNjNfj4gvA1+MiH8GfgiclZlPRMSfAIuwBa8AAAKjSURBVL8HfguMy8ztEXEC8APAD/mTymTmlIg4FRidmS+3dD3vRgaZ1ufSiJhQ2j6O+h+a9uYU4DyA0hSbLYe4Nulw2pCZj5W2/x24tLT9Q4DSb4uHA/9RPysGgPal/+7v2XgC+F5EtAN+nJk1jdpPAu7KzNdL97oTqKb+t2q/Luv/JFB1EK9RamkvZOayiDgN6AM8Vnqe3gP8lPqgs6n0m2Qy8zWAiHgv8H8ioj/wJtCzJYqXdGQzyLQiETEKGAsMy8w3ImIx0AEof4/sDi1QmtQSGr83/O7910v/PQp4NTP7H/CFMx+JiJHAX1H/Yb7/kpnzy7rEXk4F+EPZ9puAU8tUZLufpwAWZuak8saI6MeezyLAdOBF4GPUP4vbD2WRktQU18i0Lp2BzaUQ0wsYWjr+YkT0joijgAll/R8EpkL9wufSkL/0bnF8RAwrbU8CHi1vLP1m+NcRMRHqFytHxMdKzft8NiLiQ8BvM/NfgTnAwEb3fgQ4s7Qu573UP3dLm++lSa3OMmBERPwFQOnvfk9gLfDBiPhE6XiniGhL/b9Xm0pvDnAu0KaF6pZ0BDPItC73A20j4ingWur/YQH4CvAT4CFgU1n//wmMjoinqZ/icuJhrFU61J4FJpeehw8A/7eJPp8HvhARPwdWA2eUju/v2RgF1ETEKurXuHyzvDEzVwJzgeXAz4B/y8xVzfCapFYpM18Czgd+UHrmlgG9MnMHcBbwv0vP2ULqZwZ8m/rncxn108peb/LCkgCIiD+LiFrgi8A/REStv4A+eJHZ1IixJLWciKgCfpKZH23hUiRJUivliIwkSZKkwnFERpIkSVLhOCIjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIK5/8DFExRO+5ZjHUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PERFORMANCE ON THE STANDARD TEST ###\n",
    "\n",
    "plt.figure(figsize=(14,5))\n",
    "plt.bar(np.arange(len(rf_train_aug))-0.1, rf_train_aug.values(), \n",
    "        alpha=0.5, width=0.2, label='w/ augmentation')\n",
    "plt.bar(np.arange(len(rf_train))+0.1, rf_train.values(), \n",
    "        alpha=0.5, width=0.2, label='w/o augmentation')\n",
    "plt.xticks(range(len(rf_train_aug)), rf_train_aug.keys())\n",
    "plt.title('Random Forest'); plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAE/CAYAAACdNPbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df5xWdZ338ddHINgNoxLWVdAGdyVAISDip4Og0lorKm2kbCpKt964i943bVm6m+GPbtN4iGtbS+2aiOtmblppD3+BCuIPQoRJBTHJMGZxTQ1RNITRz/3HXEyXwwAXcsHMgdfz8ZiH55zv93zP52I4Mu/5fs91RWYiSZIkSUWyX2sXIEmSJEk7yyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpIkSZIKxyAjSZIkqXAMMpKkXRYRoyOivrXrkCTtOwwykrSXiojVEfGHiNgQEf8TEbMjonNr17WrIiIj4o3S69oQEa/u4esb2iSpDTDISNLebVxmdgYGAAOBC1u5nmr5WGZ2Ln19cGdPjoj2u6MoSdKeY5CRpH1AZv4PcA+NgQaAiPjriFgWEa9FxJqImF7WVlOa+ZgUEb+NiJcj4h/L2v+kNMOzLiJWAJ8ov15E9ImI+RHxakQsj4gTy9pmR8R3I+Ku0ozKwxHx5xFxTWm8lREx8L28zog4OyJWRcTvI+L2iDi4rC0j4u8j4lng2dKx3hExt9T/mYj4XFn/T0fEioh4PSL+OyK+FBHvB+4CDi6bETp4q0IkSbudQUaS9gER0QP4FLCq7PAbwBnAB4G/Bs6NiJObnXoU8FHgWODiiOhTOv514C9KX38FTCq7VgfgDuBe4M+A84CbIuKjZeN+DvgnoCvwFvAosLS0/2Pg6vfwGo8BriiNfRDwPHBzs24nA0OBvqVQMhf4z1KdE4HvRsQRpb7XAf87M/cHjgTuz8w3aPxzXFs2I7R2Z2uVJO06g4wk7d1+GhGvA2uA39EYQADIzPmZ+WRmvpOZTwA/BI5udv4lmfmHzPwl8EvgY6XjnwO+kZm/z8w1wLVl5wwDOgPfzMxNmXk/8HMag8IWP8nMxzNzI/ATYGNmzsnMt4Ef0bgMbnuWlmZ7Xo2ILdf+PPCDzFyamW/RuIxueETUlJ13RanmPwAnAKsz8/rMbMjMpcCtwGdLfTfTGHg+kJnrSu2SpDbCICNJe7eTSzMKo4HeNM54ABARQyPigYh4KSLWA1PK20v+p2z7TRoDCsDBNIajLZ4v2z4YWJOZ7zRr7162/2LZ9h9a2N/RmxIMyswPlr7OL7tuUx2ZuQF4pdl1y2v+CDC0LBC9SmMY+vNS+98Anwaej4gFETF8BzVJkvYgg4wk7QMycwEwG5hRdvg/gduBQzKzCzALiAqHfAE4pGz/0LLttcAhEbFfs/b/3smyd9ZaGsMJAKWlYwc0u26Wba8BFpQFog+WloqdC5CZj2XmSTQuO/spcEsLY0iSWolBRpL2HdcAYyNiywP/+wO/z8yNETEE+NudGOsW4MKI+FDp+Zvzytp+QePzNxdERIeIGA2MY+vnVartP4GzImJARHQE/h/wi8xcvY3+Pwd6RcTppTo7RMQnSm9U8L6I+HxEdMnMzcBrwNul814EDoiILrv59UiStsMgI0n7iMx8CZgDfK106O+AS0vP0FzMH2ccKnEJjcu4fkPjQ/03ll1nE3AijQ/Fvwx8FzgjM1fu6mvYnsy8j8bXdiuNM0Z/AZy6nf6vA58s9VlL4zK6K4GOpS6nA6sj4jUal92dVjpvJY3PEz1XWpLmu5ZJUiuITGfIJUmSJBWLMzKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCqd9a124a9euWVNT01qXlyRJktTGPf744y9nZreW2lotyNTU1LBkyZLWurwkSZKkNi4int9Wm0vLJEmSJBWOQUaSJElS4RhkJEmSJBXODp+RiYgfACcAv8vMI1toD+CfgU8DbwJnZubSahcqSZKkfcPmzZupr69n48aNrV2K9pBOnTrRo0cPOnToUPE5lTzsPxv4F2DONto/BRxe+hoK/Gvpv5IkSdJOq6+vZ//996empobG35lrb5aZvPLKK9TX19OzZ8+Kz9vh0rLMfBD4/Xa6nATMyUaLgA9GxEEVVyBJkiSV2bhxIwcccIAhZh8RERxwwAE7PQNXjWdkugNryvbrS8e2EhHnRMSSiFjy0ksvVeHSkiRJ2hsZYvYt7+X7XY0g09JVs6WOmfn9zBycmYO7dWvxc20kSZKkQrjiiiu46aabWruMnTZ//nweeeSRne43a9Ys5szZ1tMme141PhCzHjikbL8HsLYK40qSJEnMnPurqo43bWyvqoxz7733csstt1RlrD1p/vz5dO7cmREjRuxUvylTpuyJ8ipWjRmZ24EzotEwYH1mvlCFcSVJkqQ97qqrruLaa68FYNq0aRxzzDEA3HfffZx22mkAvPbaa2zatInmq4wWL17MiBEjGDhwICNGjOCZZ54BYPbs2UydOrWp3wknnMD8+fMBuO666+jVqxejR4/m7LPPbup35plncu655zJmzBgOO+wwFixYwOTJk+nTpw9nnnlm01j33nsvw4cPZ9CgQUyYMIENGzYAUFNTw9e//nUGDRpEv379WLlyJatXr2bWrFnMnDmTAQMGsHDhQu644w6GDh3KwIEDOe6443jxxRdb7Dd9+nRmzJgBQF1dHcOGDaN///6MHz+edevWATB69Gi+8pWvMGTIEHr16sXChQur+a15lx0GmYj4IfAo8NGIqI+IL0TElIjYEsnuBJ4DVgH/BvzdbqtWkiRJ2s1GjRrV9AP4kiVL2LBhA5s3b+ahhx6itrYWgHnz5nHsscdudW7v3r158MEHWbZsGZdeeikXXXTRdq+1du1aLrvsMhYtWsTcuXNZuXLlu9rXrVvH/fffz8yZMxk3bhzTpk1j+fLlPPnkk9TV1fHyyy9z+eWXM2/ePJYuXcrgwYO5+uqrm87v2rUrS5cu5dxzz2XGjBnU1NQwZcoUpk2bRl1dHbW1tRx11FEsWrSIZcuWceqpp3LVVVe12K/cGWecwZVXXskTTzxBv379uOSSS5raGhoaWLx4Mddcc827jlfbDpeWZebEHbQn8PdVq6iVVHvKcneo1jSoJEmStu3jH/84jz/+OK+//jodO3Zk0KBBLFmyhIULFzbN1Nx9992cddZZW527fv16Jk2axLPPPktEsHnz5u1ea/HixRx99NF8+MMfBmDChAn86ld//Ll03LhxRAT9+vXjwAMPpF+/fgAcccQRrF69mvr6elasWMHIkSMB2LRpE8OHD286/zOf+UzTa7rttttarKG+vp5TTjmFF154gU2bNu3wLZDXr1/Pq6++ytFHHw3ApEmTmDBhQovXXL169XbH2hXVWFomSZIk7TU6dOhATU0N119/PSNGjKC2tpYHHniAX//61/Tp0wdoDCBDhgzZ6tyvfe1rjBkzhqeeeoo77rij6S2F27dvzzvvvNPUb8vxxjmBbevYsSMA++23X9P2lv2GhgYyk7Fjx1JXV0ddXR0rVqzguuuu2+r8du3a0dDQ0OI1zjvvPKZOncqTTz7J9773vV3+INJKrlkNBhlJkiSpmVGjRjFjxgxGjRpFbW0ts2bNYsCAAUQEy5cvp3fv3rRr126r89avX0/37o2fRDJ79uym4zU1NdTV1fHOO++wZs0aFi9eDMCQIUNYsGAB69ato6GhgVtvvXWn6hw2bBgPP/wwq1atAuDNN99814xOS/bff39ef/31Fmu+4YYbttlviy5duvChD32oafndjTfe2DQ7sycZZCRJkqRmamtreeGFFxg+fDgHHnggnTp1anpO5K677uL4449v8bwLLriACy+8kJEjR/L22283HR85ciQ9e/akX79+fOlLX2LQoEEAdO/enYsuuoihQ4dy3HHH0bdvX7p06VJxnd26dWP27NlMnDiR/v37M2zYsK2es2lu3Lhx/OQnP3nXQ/wTJkygtraWrl27brNfuRtuuIEvf/nL9O/fn7q6Oi6++OKKa66W2NF01u4yePDgXLJkSatcuyU+IyNJktQ2PP30001LuNqisWPHMmfOHA466KCqjLdhwwY6d+5MQ0MD48ePZ/LkyYwfP74qYxdJS9/3iHg8Mwe31L8anyMjSZIk7TPmzp1b1fGmT5/OvHnz2LhxI5/85Cc5+eSTqzr+3sogI0mSJLWiLZ/Nop3jMzKSJEmSCscgI0mSJKlwXFpWJA9c0doVVGbMha1dgSRJkvZyzshIkiRJKhyDjCRJkvQeXHHFFdx0002tXUbVzJ8/n0ceeWSn+82aNYs5c+bsztJa5NIySZIktW3VXl5fpWXw9957L7fccktVxmoL5s+fT+fOnRkxYsRO9ZsyZcqeKG8rzshIkiRJZa666iquvfZaAKZNm8YxxxwDwH333cdpp50GwGuvvcamTZvo1q0bzz//PMceeyz9+/fn2GOP5be//e1WYy5evJgRI0YwcOBARowYwTPPPAPA7NmzmTp1alO/E044gfnz5wNw3XXX0atXL0aPHs3ZZ5/d1O/MM8/k3HPPZcyYMRx22GEsWLCAyZMn06dPH84888ymse69916GDx/OoEGDmDBhAhs2bACgpqaGr3/96wwaNIh+/fqxcuVKVq9ezaxZs5g5cyYDBgxg4cKF3HHHHQwdOpSBAwdy3HHH8eKLL7bYb/r06U1vIV1XV8ewYcPo378/48ePZ926dQCMHj2ar3zlKwwZMoRevXqxcOHCXf4+GWQkSZKkMqNGjWr6QXvJkiVs2LCBzZs389BDD1FbWwvAvHnzOPbYYwGYOnUqZ5xxBk888QSf//znOf/887cas3fv3jz44IMsW7aMSy+9lIsuumi7Naxdu5bLLruMRYsWMXfuXFauXPmu9nXr1nH//fczc+ZMxo0bx7Rp01i+fDlPPvkkdXV1vPzyy1x++eXMmzePpUuXMnjwYK6++uqm87t27crSpUs599xzmTFjBjU1NUyZMoVp06ZRV1dHbW0tRx11FIsWLWLZsmWceuqpXHXVVS32K3fGGWdw5ZVX8sQTT9CvXz8uueSSpraGhgYWL17MNddc867j75VLyyRJkqQyH//4x3n88cd5/fXX6dixI4MGDWLJkiUsXLiwaabm7rvv5qyzzgLg0Ucf5bbbbgPg9NNP54ILLthqzPXr1zNp0iSeffZZIoLNmzdvt4bFixdz9NFH8+EPfxiACRMm8Ktf/aqpfdy4cUQE/fr148ADD6Rfv34AHHHEEaxevZr6+npWrFjByJEjAdi0aRPDhw9vOv8zn/lM02vdUntz9fX1nHLKKbzwwgts2rSJnj17brfm9evX8+qrr3L00UcDMGnSJCZMmNDiNVevXr3dsSrhjIwkSZJUpkOHDtTU1HD99dczYsQIamtreeCBB/j1r39Nnz59gMagMWTIkBbPj4itjn3ta19jzJgxPPXUU9xxxx1s3LgRgPbt2/POO+809dtyPDO3W2PHjh0B2G+//Zq2t+w3NDSQmYwdO5a6ujrq6upYsWIF11133Vbnt2vXjoaGhhavcd555zF16lSefPJJvve97zXV9l5Vcs2dYZCRJEmSmhk1ahQzZsxg1KhR1NbWMmvWLAYMGEBEsHz5cnr37k27du0AGDFiBDfffDMAN910E0cdddRW461fv57u3bsDjc/FbFFTU0NdXR3vvPMOa9asYfHixQAMGTKEBQsWsG7dOhoaGrj11lt3qv5hw4bx8MMPs2rVKgDefPPNd83otGT//ffn9ddfb7HmG264YZv9tujSpQsf+tCHmpbl3XjjjU2zM7uDS8sk7bKZc7f/P8a2YtrYXq1dgiSpIGpra/nGN77B8OHDef/730+nTp2ange56667OP7445v6XnvttUyePJlvfetbdOvWjeuvv36r8S644AImTZrE1Vdf3fTmAQAjR46kZ8+e9OvXjyOPPJJBgwYB0L17dy666CKGDh3KwQcfTN++fenSpUvF9Xfr1o3Zs2czceJE3nrrLQAuv/xyevXa9r+F48aN47Of/Sw/+9nP+Pa3v8306dOZMGEC3bt3Z9iwYfzmN79psV+5G264gSlTpvDmm29y2GGHtfhnUS2xo2mr3WXw4MG5ZMmSVrl2S4rwg9i09juXxFtNld7SUMVRhPsHDDKSVBRPP/100xKutmjs2LHMmTOHgw46aLdeZ8OGDXTu3JmGhgbGjx/P5MmTGT9+/G69Zmtq6fseEY9n5uCW+jsjI0mSJO2EuXPn7pHrTJ8+nXnz5rFx40Y++clPcvLJJ++R6xaFQUaSJElqg7Z8NotaZpCRJKmVuTxT2jUvvrZr76a1pxz4gU6tXcJexXctkyRJUpvTWs9xq3W8l++3QUaSJEltSqdOnXjllVcMM/uIzOSVV16hU6edm7FyaZkkSZLalB49elBfX89LL71UUf/X/rB5N1dUHb//kw6tXUKb1alTJ3r06LFT5xhkJEmS1KZ06NCBnj17Vtzf58z2TS4tkyRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4FQWZiDg+Ip6JiFUR8dUW2g+NiAciYllEPBERn65+qZIkSZLUaIdBJiLaAd8BPgX0BSZGRN9m3f4JuCUzBwKnAt+tdqGSJEmStEUlMzJDgFWZ+VxmbgJuBk5q1ieBD5S2uwBrq1eiJEmSJL1b+wr6dAfWlO3XA0Ob9ZkO3BsR5wHvB46rSnWSJEmS1IJKZmSihWPZbH8iMDszewCfBm6MiK3GjohzImJJRCx56aWXdr5aSZIkSaKyIFMPHFK234Otl459AbgFIDMfBToBXZsPlJnfz8zBmTm4W7du761iSZIkSfu8SoLMY8DhEdEzIt5H48P8tzfr81vgWICI6ENjkHHKRZIkSdJuscMgk5kNwFTgHuBpGt+dbHlEXBoRJ5a6/QNwdkT8EvghcGZmNl9+JkmSJElVUcnD/mTmncCdzY5dXLa9AhhZ3dIkSZIkqWUVfSCmJEmSJLUlBhlJkiRJhWOQkSRJklQ4BhlJkiRJhWOQkSRJklQ4Fb1rmSTtFR64orUrqMyYC1u7AkmS2jxnZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuEYZCRJkiQVjkFGkiRJUuG0r6RTRBwP/DPQDvj3zPxmC30+B0wHEvhlZv5tFeuUJEmt7YErWruCyoy5sLUrkLQH7DDIREQ74DvAWKAeeCwibs/MFWV9DgcuBEZm5rqI+LPdVbAkSZIkVbK0bAiwKjOfy8xNwM3ASc36nA18JzPXAWTm76pbpiRJkiT9USVBpjuwpmy/vnSsXC+gV0Q8HBGLSkvRJEmSJGm3qOQZmWjhWLYwzuHAaKAHsDAijszMV981UMQ5wDkAhx566E4XK0mSJElQ2YxMPXBI2X4PYG0LfX6WmZsz8zfAMzQGm3fJzO9n5uDMHNytW7f3WrMkSZKkfVwlQeYx4PCI6BkR7wNOBW5v1uenwBiAiOhK41Kz56pZqCRJkiRtscMgk5kNwFTgHuBp4JbMXB4Rl0bEiaVu9wCvRMQK4AHgy5n5yu4qWpIkSdK+raLPkcnMO4E7mx27uGw7gS+WviRJkiRpt6pkaZkkSZIktSkGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgGGUmSJEmFY5CRJEmSVDgVfY6MJEmSpF30wBWtXUFlxlzY2hVUxBkZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOAYZSZIkSYVjkJEkSZJUOBUFmYg4PiKeiYhVEfHV7fT7bERkRAyuXomSJEmS9G47DDIR0Q74DvApoC8wMSL6ttBvf+B84BfVLlKSJEmSylUyIzMEWJWZz2XmJuBm4KQW+l0GXAVsrGJ9kiRJkrSVSoJMd2BN2X596ViTiBgIHJKZP9/eQBFxTkQsiYglL7300k4XK0mSJElQWZCJFo5lU2PEfsBM4B92NFBmfj8zB2fm4G7dulVepSRJkiSVqSTI1AOHlO33ANaW7e8PHAnMj4jVwDDgdh/4lyRJkrS7VBJkHgMOj4ieEfE+4FTg9i2Nmbk+M7tmZk1m1gCLgBMzc8luqViSJEnSPm+HQSYzG4CpwD3A08Atmbk8Ii6NiBN3d4GSJEmS1Fz7Sjpl5p3Anc2OXbyNvqN3vSxJkiRJ2raKPhBTkiRJktoSg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSocg4wkSZKkwjHISJIkSSqcioJMRBwfEc9ExKqI+GoL7V+MiBUR8URE3BcRH6l+qZIkSZLUaIdBJiLaAd8BPgX0BSZGRN9m3ZYBgzOzP/Bj4KpqFypJkiRJW1QyIzMEWJWZz2XmJuBm4KTyDpn5QGa+WdpdBPSobpmSJEmS9EeVBJnuwJqy/frSsW35AnDXrhQlSZIkSdvTvoI+0cKxbLFjxGnAYODobbSfA5wDcOihh1ZYoiRJkiS9WyUzMvXAIWX7PYC1zTtFxHHAPwInZuZbLQ2Umd/PzMGZObhbt27vpV5JkiRJqijIPAYcHhE9I+J9wKnA7eUdImIg8D0aQ8zvql+mJEmSJP3RDoNMZjYAU4F7gKeBWzJzeURcGhEnlrp9C+gM/FdE1EXE7dsYTpIkSZJ2WSXPyJCZdwJ3Njt2cdn2cVWuS5IkSZK2qaIPxJQkSZKktsQgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwDDKSJEmSCscgI0mSJKlwKgoyEXF8RDwTEasi4qsttHeMiB+V2n8RETXVLlSSJEmStthhkImIdsB3gE8BfYGJEdG3WbcvAOsy8y+BmcCV1S5UkiRJkraoZEZmCLAqM5/LzE3AzcBJzfqcBNxQ2v4xcGxERPXKlCRJkqQ/qiTIdAfWlO3Xl4612CczG4D1wAHVKFCSJEmSmovM3H6HiAnAX2Xm/yrtnw4MyczzyvosL/WpL+3/utTnlWZjnQOcU9r9KPBMtV7IPqIr8HJrFyEVmPeQtGu8h6Rd4z208z6Smd1aamhfwcn1wCFl+z2AtdvoUx8R7YEuwO+bD5SZ3we+X0nF2lpELMnMwa1dh1RU3kPSrvEeknaN91B1VbK07DHg8IjoGRHvA04Fbm/W53ZgUmn7s8D9uaOpHkmSJEl6j3Y4I5OZDRExFbgHaAf8IDOXR8SlwJLMvB24DrgxIlbROBNz6u4sWpIkSdK+rZKlZWTmncCdzY5dXLa9EZhQ3dLUApflSbvGe0jaNd5D0q7xHqqiHT7sL0mSJEltTSXPyEiSJElSm2KQkbTPiIhHdtB+Z0R8cE/VI+1tIqImIp4qbY+OiJ+3dk1Sa4qI8yPi6Yi4NSIejYi3IuJLrV3X3qKiZ2Qkqa2JiHaZ+fbOnJOZI3bQ/uldq0oqpogIGpebv9PatUh7mb8DPgW8AXwEOLl1y9m7OCPTxkTETyPi8YhYXvoAUSJiQ1n7ZyNidmn7wIj4SUT8svS13R/SpKIo/VZ3ZUTcEBFPRMSPI+JPI2J1RFwcEQ8BEyLiLyLi7tI9szAiepfOb/He2HIvRcRBEfFgRNRFxFMRUVs6vjoiupa2v1hqeyoi/m9ZXU9HxL+V7tF7I7PQuugAAAN6SURBVOJPWuUPSdpFZX+fvwssBU4v/cZ4aUT8V0R0LvX7REQ8UrqXFkfE/qVzF5b6LvXfH2lrETELOIzGjyn5fGY+Bmxu3ar2Ls7ItD2TM/P3pR+OHouIW7fT91pgQWaOj4h2QOc9U6K0R3wU+EJmPhwRP6Dxt1oAGzPzKICIuA+YkpnPRsRQ4LvAMez43vhb4J7M/Eap/U/LGyPi48BZwFAggF9ExAJgHXA4MDEzz46IW4C/Af6j6q9e2jM+SuPf9YuB24DjMvONiPgK8MWI+CbwI+CUzHwsIj4A/AH4HTA2MzdGxOHADwE/5E8qk5lTIuJ4YExmvtza9eyNDDJtz/kRMb60fQiNPzRtyzHAGQClJTbrd3Nt0p60JjMfLm3/B3B+aftHAKXfFo8A/qtxVQwAHUv/3dG98Rjwg4joAPw0M+uatR8F/CQz3yhd6zaglsbfqv2mrP/jQM0uvEaptT2fmYsi4gSgL/Bw6X56H/AojUHnhdJvksnM1wAi4v3Av0TEAOBtoFdrFC9p32aQaUMiYjRwHDA8M9+MiPlAJ6D8PbI7tUJpUmto/t7wW/bfKP13P+DVzByw0wNnPhgRo4C/pvHDfL+VmXPKusQ2TgV4q2z7bcClZSqyLfdTAHMzc2J5Y0T0Z+t7EWAa8CLwMRrvxY27s0hJaonPyLQtXYB1pRDTGxhWOv5iRPSJiP2A8WX97wPOhcYHn0tT/tLe4tCIGF7angg8VN5Y+s3wbyJiAjQ+rBwRHys1b/feiIiPAL/LzH8DrgMGNbv2g8DJpedy3k/jfbewei9NanMWASMj4i8BSn/3ewErgYMj4hOl4/tHRHsa/716ofTmAKcD7Vqpbkn7MINM23I30D4ingAuo/EfFoCvAj8H7gdeKOv/f4AxEfEkjUtcjtiDtUq729PApNL98GHgX1vo83ngCxHxS2A5cFLp+I7ujdFAXUQso/EZl38ub8zMpcBsYDHwC+DfM3NZFV6T1CZl5kvAmcAPS/fcIqB3Zm4CTgG+XbrP5tK4MuC7NN6fi2hcVvZGiwNLAiAi/jwi6oEvAv8UEfX+AnrXRWZLM8aS1Hoiogb4eWYe2cqlSJKkNsoZGUmSJEmF44yMJEmSpMJxRkaSJElS4RhkJEmSJBWOQUaSJElS4RhkJEmSJBWOQUaSJElS4RhkJEmSJBXO/wdg351Re4wbbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1008x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PERFORMANCE ON THE FAKE TEST ###\n",
    "\n",
    "plt.figure(figsize=(14,5))\n",
    "plt.bar(np.arange(len(rf_train_aug_new))-0.1, rf_train_aug_new.values(), \n",
    "        alpha=0.5, width=0.2, label='w/ augmentation')\n",
    "plt.bar(np.arange(len(rf_train_new))+0.1, rf_train_new.values(), \n",
    "        alpha=0.5, width=0.2, label='w/o augmentation')\n",
    "plt.xticks(range(len(rf_train_aug_new)), rf_train_aug_new.keys())\n",
    "plt.title('Random Forest'); plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
